← All changes | admin/class-knowledge-manager.php +1612 -6421 3.2.10 → 2.2.3 View file →

		@@ -15,72 +15,49 @@
15	15
16	16	/**
17	17	* Constructor - Register hooks for content processing
18	18	*/
19		-public function __construct() {
20		- $this->options = get_option('mxchat_options', array());
21		- $this->mxchat_init_hooks();
	19	+ public function __construct() {
	20	+ $this->options = get_option('mxchat_options', array());
	21	+ $this->mxchat_init_hooks();
	22	+ }
22	23
23		- $this->mxchat_init_role_hooks();
24		-}
25		-
26		-/**
27		- * Initialize WordPress hooks for content processing
28		- *
29		- */
30		-private function mxchat_init_hooks() {
31		- // Admin post handlers for form submissions
32		- add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
33		- add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
34		- add_action('admin_post_mxchat_submit_pdf_file', array($this, 'mxchat_handle_pdf_file_submission'));
35		- add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
36		-
37		- // AJAX handlers for real-time processing and status updates
38		- add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
39		- add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status'));
40		- add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
41		- add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
42		- add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
43		- add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
44		- add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
45		- add_action('wp_ajax_mxchat_delete_chunks_by_url', array($this, 'ajax_mxchat_delete_chunks_by_url'));
46		- add_action('wp_ajax_mxchat_delete_wordpress_prompt', array($this, 'ajax_mxchat_delete_wordpress_prompt'));
47		- add_action('wp_ajax_mxchat_bulk_delete_knowledge', array($this, 'ajax_mxchat_bulk_delete_knowledge'));
48		- add_action('wp_ajax_mxchat_update_role_restriction', array($this, 'ajax_mxchat_update_role_restriction'));
49		-
50		- // Queue-based processing AJAX handlers
51		- add_action('wp_ajax_mxchat_get_next_queue_item', array($this, 'ajax_mxchat_get_next_queue_item'));
52		- add_action('wp_ajax_mxchat_process_queue_item', array($this, 'ajax_mxchat_process_queue_item'));
53		- add_action('wp_ajax_mxchat_get_queue_status', array($this, 'ajax_mxchat_get_queue_status'));
54		- add_action('wp_ajax_mxchat_clear_queue', array($this, 'ajax_mxchat_clear_queue'));
55		- add_action('wp_ajax_mxchat_retry_failed', array($this, 'ajax_mxchat_retry_failed'));
56		- add_action('wp_ajax_mxchat_get_recent_entries', array($this, 'ajax_mxchat_get_recent_entries'));
57		- add_action('wp_ajax_mxchat_detect_sitemaps', array($this, 'ajax_mxchat_detect_sitemaps'));
58		- add_action('wp_ajax_mxchat_refresh_pinecone_entries', array($this, 'ajax_mxchat_refresh_pinecone_entries'));
59		- add_action('wp_ajax_mxchat_paginate_entries', array($this, 'ajax_mxchat_paginate_entries'));
60		- add_action('wp_ajax_mxchat_get_entry_content', array($this, 'ajax_mxchat_get_entry_content'));
61		- add_action('wp_ajax_mxchat_save_entry_content', array($this, 'ajax_mxchat_save_entry_content'));
	24	+ /**
	25	+ * Initialize WordPress hooks for content processing
	26	+ */
	27	+ private function mxchat_init_hooks() {
	28	+ // Admin post handlers for form submissions
	29	+ add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
	30	+ add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
	31	+ add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
	32	+
	33	+ // AJAX handlers for real-time processing and status updates
	34	+ add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
	35	+ add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status')); // NEW
	36	+ add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
	37	+ add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
	38	+ add_action('wp_ajax_mxchat_manual_batch_process', array($this, 'ajax_manual_batch_process'));
	39	+ add_action('mxchat_delete_content', array($this, 'mxchat_delete_from_pinecone_by_url'), 10, 1);
62	40
63		- // WordPress post management hooks
64		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
65		- add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
66		- add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
67		- add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
	41	+
	42	+ // Cron handlers for background processing
	43	+ add_action('mxchat_process_sitemap_urls', array($this, 'mxchat_process_sitemap_urls_cron'), 10, 5);
	44	+ add_action('mxchat_process_pdf_pages', array($this, 'mxchat_process_pdf_pages_cron'), 10, 5);
	45	+
	46	+ // WordPress post management hooks
	47	+ add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
	48	+ add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
	49	+ add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
	50	+ add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
68	51
69		- // ACF hook - fires AFTER ACF fields are saved, ensuring ACF data is available
70		- // Priority 20 to run after ACF's own save (which runs at priority 10)
71		- add_action('acf/save_post', array($this, 'mxchat_handle_acf_save'), 20);
72		-
73		- add_action('wp_ajax_mxchat_mark_queue_complete', array($this, 'ajax_mxchat_mark_queue_complete'));
74		-
75		- // WooCommerce product hooks (if WooCommerce is active)
76		- if (class_exists('WooCommerce')) {
77		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
78		- add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
79		- add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
80		- add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
	52	+ // WooCommerce product hooks (if WooCommerce is active)
	53	+ if (class_exists('WooCommerce')) {
	54	+ add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
	55	+ add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
	56	+ add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
	57	+ }
	58	+
81	59	}
82		-}
83	60
84	61	/**
85	62	* Get current options (refreshed)
86	63	*/
		@@ -90,9 +67,315 @@
90	67	}
91	68	return $this->options;
92	69	}
93	70
	71	+
	72	+ /**
	73	+ * Handle manual batch processing via AJAX
	74	+ */
	75	+public function ajax_manual_batch_process() {
	76	+ try {
	77	+ // Verify nonce and permissions
	78	+ check_ajax_referer('mxchat_status_nonce', 'nonce');
	79	+
	80	+ if (!current_user_can('manage_options')) {
	81	+ wp_send_json_error('Unauthorized access');
	82	+ }
	83	+
	84	+ $process_type = sanitize_text_field($_POST['process_type'] ?? '');
	85	+ $url = sanitize_text_field($_POST['url'] ?? '');
	86	+
	87	+ if (empty($process_type) || empty($url)) {
	88	+ wp_send_json_error('Missing required parameters');
	89	+ }
	90	+
	91	+ $processed = 0;
	92	+
	93	+ if ($process_type === 'pdf') {
	94	+ $processed = $this->mxchat_manual_process_pdf_batch($url);
	95	+ } elseif ($process_type === 'sitemap') {
	96	+ $processed = $this->mxchat_manual_process_sitemap_batch($url);
	97	+ }
	98	+
	99	+ if ($processed > 0) {
	100	+ wp_send_json_success(array(
	101	+ 'message' => "Processed {$processed} items successfully",
	102	+ 'processed' => $processed
	103	+ ));
	104	+ } else {
	105	+ wp_send_json_error('No items were processed');
	106	+ }
	107	+
	108	+ } catch (Exception $e) {
	109	+ //error_log('Manual batch process error: ' . $e->getMessage());
	110	+ wp_send_json_error('Processing failed: ' . $e->getMessage());
	111	+ }
	112	+}
94	113
	114	+/**
	115	+ * Process a small PDF batch manually - DIRECT PROCESSING
	116	+ */
	117	+private function mxchat_manual_process_pdf_batch($pdf_url) {
	118	+ try {
	119	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	120	+ $status = get_transient($status_key);
	121	+
	122	+ if (!$status || $status['status'] !== 'processing') {
	123	+ //error_log('Manual PDF: No processing status found');
	124	+ return 0;
	125	+ }
	126	+
	127	+ //error_log('Manual PDF: Starting direct processing for ' . $pdf_url);
	128	+
	129	+ // Get current progress
	130	+ $current_page = $status['processed_pages'] ?? 0;
	131	+ $total_pages = $status['total_pages'] ?? 0;
	132	+
	133	+ if ($current_page >= $total_pages) {
	134	+ //error_log('Manual PDF: Already completed');
	135	+ return 0;
	136	+ }
	137	+
	138	+ // Try to download the PDF again for processing
	139	+ $response = wp_remote_get($pdf_url, array('timeout' => 30));
	140	+
	141	+ if (is_wp_error($response)) {
	142	+ //error_log('Manual PDF: Failed to download PDF: ' . $response->get_error_message());
	143	+ return 0;
	144	+ }
	145	+
	146	+ $pdf_content = wp_remote_retrieve_body($response);
	147	+ if (empty($pdf_content)) {
	148	+ //error_log('Manual PDF: Empty PDF content');
	149	+ return 0;
	150	+ }
	151	+
	152	+ // Save PDF temporarily
	153	+ $upload_dir = wp_upload_dir();
	154	+ $temp_pdf_path = trailingslashit($upload_dir['path']) . 'temp_manual_' . time() . '.pdf';
	155	+ file_put_contents($temp_pdf_path, $pdf_content);
	156	+
	157	+ // Process 2 pages directly
	158	+ $processed = $this->mxchat_process_pdf_pages_direct($temp_pdf_path, $pdf_url, $current_page, 5);
	159	+
	160	+ // Clean up temp file
	161	+ if (file_exists($temp_pdf_path)) {
	162	+ wp_delete_file($temp_pdf_path);
	163	+ }
	164	+
	165	+ //error_log('Manual PDF: Processed ' . $processed . ' pages');
	166	+ return $processed;
	167	+
	168	+ } catch (Exception $e) {
	169	+ //error_log('Manual PDF batch error: ' . $e->getMessage());
	170	+ return 0;
	171	+ }
	172	+}
	173	+
	174	+/**
	175	+ * Process PDF pages directly without cron
	176	+ */
	177	+private function mxchat_process_pdf_pages_direct($pdf_path, $pdf_url, $start_page, $batch_size) {
	178	+ try {
	179	+ if (!file_exists($pdf_path)) {
	180	+ //error_log('Direct PDF: File not found at ' . $pdf_path);
	181	+ return 0;
	182	+ }
	183	+
	184	+ $parser = new \Smalot\PdfParser\Parser();
	185	+ $pdf = $parser->parseFile($pdf_path);
	186	+ $pages = $pdf->getPages();
	187	+
	188	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	189	+ $status = get_transient($status_key);
	190	+
	191	+ if (!$status) {
	192	+ return 0;
	193	+ }
	194	+
	195	+ $options = get_option('mxchat_options');
	196	+ $api_key = $options['api_key'] ?? '';
	197	+
	198	+ if (empty($api_key)) {
	199	+ //error_log('Direct PDF: No API key');
	200	+ return 0;
	201	+ }
	202	+
	203	+ $processed = 0;
	204	+ $end_page = min($start_page + $batch_size, count($pages));
	205	+
	206	+ for ($i = $start_page; $i < $end_page; $i++) {
	207	+ try {
	208	+ $page_number = $i + 1;
	209	+ $text = $pages[$i]->getText();
	210	+
	211	+ if (empty($text)) {
	212	+ //error_log('Direct PDF: Empty text on page ' . $page_number);
	213	+ continue;
	214	+ }
	215	+
	216	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
	217	+ if (empty($sanitized_content)) {
	218	+ //error_log('Direct PDF: No content after sanitization on page ' . $page_number);
	219	+ continue;
	220	+ }
	221	+
	222	+ // Generate embedding
	223	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	224	+ if (is_string($embedding_vector)) {
	225	+ //error_log('Direct PDF: Embedding failed on page ' . $page_number . ': ' . $embedding_vector);
	226	+ continue;
	227	+ }
	228	+
	229	+ // Create metadata
	230	+ $metadata = array(
	231	+ 'document_type' => 'pdf',
	232	+ 'total_pages' => count($pages),
	233	+ 'current_page' => $page_number,
	234	+ 'source_url' => $pdf_url
	235	+ );
	236	+
	237	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	238	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
	239	+
	240	+ // Store in database
	241	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $api_key);
	242	+
	243	+ if (is_wp_error($db_result)) {
	244	+ //error_log('Direct PDF: DB error on page ' . $page_number . ': ' . $db_result->get_error_message());
	245	+ continue;
	246	+ }
	247	+
	248	+ $processed++;
	249	+ //error_log('Direct PDF: Successfully processed page ' . $page_number);
	250	+
	251	+ // Update status
	252	+ $status['processed_pages'] = $i + 1;
	253	+ $status['last_update'] = time();
	254	+ $status['percentage'] = round(($status['processed_pages'] / $status['total_pages']) * 100);
	255	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	256	+
	257	+ } catch (Exception $e) {
	258	+ //error_log('Direct PDF: Error processing page ' . ($i + 1) . ': ' . $e->getMessage());
	259	+ continue;
	260	+ }
	261	+ }
	262	+
	263	+ // Check if completed
	264	+ if ($status['processed_pages'] >= $status['total_pages']) {
	265	+ $status['status'] = 'complete';
	266	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	267	+ //error_log('Direct PDF: Processing completed');
	268	+ }
	269	+
	270	+ return $processed;
	271	+
	272	+ } catch (Exception $e) {
	273	+ //error_log('Direct PDF processing error: ' . $e->getMessage());
	274	+ return 0;
	275	+ }
	276	+}
	277	+
	278	+/**
	279	+ * Process a small sitemap batch manually - DIRECT PROCESSING
	280	+ */
	281	+private function mxchat_manual_process_sitemap_batch($sitemap_url) {
	282	+ try {
	283	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	284	+ $status = get_transient($status_key);
	285	+
	286	+ if (!$status || $status['status'] !== 'processing') {
	287	+ return 0;
	288	+ }
	289	+
	290	+ //error_log('Manual Sitemap: Starting direct processing for ' . $sitemap_url);
	291	+
	292	+ // Re-fetch the sitemap to get URLs
	293	+ $response = wp_remote_get($sitemap_url, array('timeout' => 30));
	294	+ if (is_wp_error($response)) {
	295	+ //error_log('Manual Sitemap: Failed to fetch sitemap');
	296	+ return 0;
	297	+ }
	298	+
	299	+ $sitemap_content = wp_remote_retrieve_body($response);
	300	+ $xml = simplexml_load_string($sitemap_content);
	301	+
	302	+ if (!$xml) {
	303	+ //error_log('Manual Sitemap: Invalid XML');
	304	+ return 0;
	305	+ }
	306	+
	307	+ $urls = array();
	308	+ foreach ($xml->url as $url_element) {
	309	+ $urls[] = (string)$url_element->loc;
	310	+ }
	311	+
	312	+ $current_processed = $status['processed_urls'] ?? 0;
	313	+ $batch_size = 5;
	314	+ $processed = 0;
	315	+
	316	+ // Process next 2 URLs
	317	+ for ($i = $current_processed; $i < min($current_processed + $batch_size, count($urls)); $i++) {
	318	+ $url = $urls[$i];
	319	+
	320	+ if ($this->mxchat_process_single_url_direct($url)) {
	321	+ $processed++;
	322	+ }
	323	+
	324	+ // Update status
	325	+ $status['processed_urls'] = $i + 1;
	326	+ $status['last_update'] = time();
	327	+ $status['percentage'] = round(($status['processed_urls'] / $status['total_urls']) * 100);
	328	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	329	+ }
	330	+
	331	+ // Check if completed
	332	+ if ($status['processed_urls'] >= $status['total_urls']) {
	333	+ $status['status'] = 'complete';
	334	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	335	+ }
	336	+
	337	+ //error_log('Manual Sitemap: Processed ' . $processed . ' URLs');
	338	+ return $processed;
	339	+
	340	+ } catch (Exception $e) {
	341	+ //error_log('Manual sitemap batch error: ' . $e->getMessage());
	342	+ return 0;
	343	+ }
	344	+}
	345	+
	346	+/**
	347	+ * Process a single URL directly
	348	+ */
	349	+private function mxchat_process_single_url_direct($url) {
	350	+ try {
	351	+ $response = wp_remote_get($url, array('timeout' => 30));
	352	+ if (is_wp_error($response)) {
	353	+ return false;
	354	+ }
	355	+
	356	+ $html = wp_remote_retrieve_body($response);
	357	+ $content = $this->mxchat_extract_main_content($html);
	358	+ $sanitized = $this->mxchat_sanitize_content_for_api($content);
	359	+
	360	+ if (empty($sanitized)) {
	361	+ return false;
	362	+ }
	363	+
	364	+ $options = get_option('mxchat_options');
	365	+ $api_key = $options['api_key'] ?? '';
	366	+
	367	+ $result = MxChat_Utils::submit_content_to_db($sanitized, $url, $api_key);
	368	+
	369	+ return !is_wp_error($result);
	370	+
	371	+ } catch (Exception $e) {
	372	+ //error_log('Single URL processing error: ' . $e->getMessage());
	373	+ return false;
	374	+ }
	375	+}
	376	+
	377	+
95	378	// ========================================
96	379	// MAIN CONTENT SUBMISSION HANDLERS
97	380	// ========================================
98	381
		@@ -108,18 +391,13 @@
108	391	wp_die(esc_html__('Nonce verification failed.', 'mxchat'));
109	392	}
110	393
111	394	// Sanitize the inputs.
112		- // Use wp_kses_post to allow safe HTML and wp_unslash to remove WordPress added slashes
113		- $article_content = wp_kses_post(wp_unslash($_POST['article_content']));
	395	+ $article_content = sanitize_textarea_field($_POST['article_content']);
114	396	$article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
115	397
116		- // Get bot_id from form submission
117		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
118		-
119		- // Get bot-specific options and API key
120		- $bot_options = $this->get_bot_options($bot_id);
121		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	398	+ // Get API key for submission
	399	+ $options = get_option('mxchat_options');
122	400	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
123	401
124	402	if (strpos($selected_model, 'voyage') === 0) {
125	403	$api_key = $options['voyage_api_key'] ?? '';
		@@ -137,10 +415,10 @@
137	415	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
138	416	exit;
139	417	}
140	418
141		- // Use centralized utility function with bot_id
142		- $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key, null, $bot_id);
	419	+ // Use centralized utility function for storage
	420	+ $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key);
143	421
144	422	if (is_wp_error($result)) {
145	423	set_transient('mxchat_admin_notice_error',
146	424	esc_html__('Error storing content: ', 'mxchat') . $result->get_error_message(),
		@@ -155,23 +433,17 @@
155	433
156	434	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
157	435	exit;
158	436	}
159		-
160	437	public function mxchat_is_pdf_url($url, $response) {
161	438	$content_type = wp_remote_retrieve_header($response, 'content-type');
162	439	$file_extension = strtolower(pathinfo($url, PATHINFO_EXTENSION));
163	440
164		- // Check Content-Disposition header for .pdf filename (Google Drive sends this)
165		- $disposition = wp_remote_retrieve_header($response, 'content-disposition');
166		- $has_pdf_disposition = ! empty($disposition) && stripos($disposition, '.pdf') !== false;
167		-
168		- return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf' || $has_pdf_disposition;
	441	+ return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf';
169	442	}
170		-
171		-
172		-public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response, $bot_id = 'default') {
	443	+public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response) {
173	444	if (!current_user_can('manage_options')) {
	445	+ //error_log(esc_html__('Unauthorized PDF processing attempt', 'mxchat'));
174	446	return false;
175	447	}
176	448
177	449	$pdf_url = esc_url_raw($pdf_url);
		@@ -177,8 +449,9 @@
177	449	$pdf_url = esc_url_raw($pdf_url);
178	450	$upload_dir = wp_upload_dir();
179	451
180	452	if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
	453	+ //error_log(sprintf(esc_html__('Upload directory error: %s', 'mxchat'), esc_html($upload_dir['error'])));
181	454	return false;
182	455	}
183	456
184	457	$pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
		@@ -185,12 +458,14 @@
185	458	$pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
186	459
187	460	$response_body = wp_remote_retrieve_body($response);
188	461	if (empty($response_body)) {
	462	+ //error_log(esc_html__('Empty PDF response body', 'mxchat'));
189	463	return false;
190	464	}
191	465
192	466	if (!wp_mkdir_p(dirname($pdf_path))) {
	467	+ //error_log(sprintf(esc_html__('Failed to create directory for PDF: %s', 'mxchat'), esc_html($pdf_path)));
193	468	return false;
194	469	}
195	470
196	471	try {
		@@ -199,693 +474,295 @@
199	474	if (!file_exists($pdf_path)) {
200	475	throw new Exception(__('Failed to save PDF file', 'mxchat'));
201	476	}
202	477
203		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
204		-
205		- if ($total_pages === false || $total_pages < 1) {
206		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
207		- }
	478	+ $parser = new \Smalot\PdfParser\Parser();
	479	+ $pdf = $parser->parseFile($pdf_path);
	480	+ $total_pages = absint(count($pdf->getPages()));
208	481
209		- // Create unique queue ID
210		- $queue_id = 'pdf_' . md5($pdf_url . time());
211		-
212		- // Create array of pages to process
213		- $pages = array();
214		- for ($i = 1; $i <= $total_pages; $i++) {
215		- $pages[] = array(
216		- 'pdf_path' => $pdf_path,
217		- 'pdf_url' => $pdf_url,
218		- 'page_number' => $i,
219		- 'total_pages' => $total_pages
220		- );
	482	+ if ($total_pages < 1) {
	483	+ throw new Exception(__('Invalid PDF: no pages found', 'mxchat'));
221	484	}
222	485
223		- // Add pages to queue
224		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
	486	+ wp_schedule_single_event(time(), 'mxchat_process_pdf_pages', array(
	487	+ 'pdf_path' => $pdf_path,
	488	+ 'pdf_url' => $pdf_url,
	489	+ 'total_pages' => $total_pages,
	490	+ 'batch_size' => absint(15),
	491	+ 'batch_pause' => absint(10)
	492	+ ));
225	493
226		- if ($queued_count === 0) {
227		- wp_delete_file($pdf_path);
228		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
229		- }
	494	+ $status_data = array(
	495	+ 'total_pages' => $total_pages,
	496	+ 'processed_pages' => 0,
	497	+ 'status' => 'processing',
	498	+ 'last_update' => time()
	499	+ );
230	500
231		- // Store queue metadata
232		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $pdf_url);
233		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
234		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
235		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
236		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
237		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	501	+ set_transient(
	502	+ sanitize_key('mxchat_pdf_status_' . md5($pdf_url)),
	503	+ array_map('sanitize_text_field', $status_data),
	504	+ DAY_IN_SECONDS
	505	+ );
238	506
239		- // Store queue ID in transient for status tracking
240		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
241		- set_transient('mxchat_last_pdf_url', $pdf_url, DAY_IN_SECONDS);
	507	+ return __('scheduled', 'mxchat');
242	508
243		- return 'queued';
244		-
245	509	} catch (Exception $e) {
	510	+ //error_log(sprintf(esc_html__('Error preparing PDF for processing: %s', 'mxchat'), esc_html($e->getMessage())));
246	511	if (file_exists($pdf_path)) {
247	512	wp_delete_file($pdf_path);
248	513	}
249		- return $e->getMessage();
	514	+ return false;
250	515	}
251	516	}
252	517
253		-/**
254		- * Handle direct PDF file upload from the knowledge base page
255		- */
256		-public function mxchat_handle_pdf_file_submission() {
257		- if (!isset($_POST['submit_pdf_file']) || !current_user_can('manage_options')) {
258		- wp_die(esc_html__('Unauthorized access', 'mxchat'));
259		- }
	518	+public function mxchat_process_pdf_pages_cron($pdf_path, $pdf_url, $total_pages, $batch_size, $batch_pause) {
	519	+ // Validate inputs
	520	+ $pdf_path = sanitize_text_field($pdf_path);
	521	+ $pdf_url = esc_url_raw($pdf_url);
	522	+ $total_pages = absint($total_pages);
	523	+ $batch_size = absint($batch_size);
	524	+ $batch_pause = absint($batch_pause);
260	525
261		- check_admin_referer('mxchat_submit_pdf_file_action', 'mxchat_submit_pdf_file_nonce');
262		-
263		- $redirect_url = admin_url('admin.php?page=mxchat-prompts');
264		-
265		- // Validate file upload
266		- if (empty($_FILES['pdf_file']) || $_FILES['pdf_file']['error'] !== UPLOAD_ERR_OK) {
267		- $error_code = isset($_FILES['pdf_file']['error']) ? $_FILES['pdf_file']['error'] : UPLOAD_ERR_NO_FILE;
268		- $error_messages = array(
269		- UPLOAD_ERR_INI_SIZE => __('The uploaded file exceeds the server upload_max_filesize limit.', 'mxchat'),
270		- UPLOAD_ERR_FORM_SIZE => __('The uploaded file exceeds the form MAX_FILE_SIZE limit.', 'mxchat'),
271		- UPLOAD_ERR_PARTIAL => __('The file was only partially uploaded.', 'mxchat'),
272		- UPLOAD_ERR_NO_FILE => __('No file was uploaded. Please select a PDF file.', 'mxchat'),
273		- UPLOAD_ERR_NO_TMP_DIR => __('Server missing temporary folder.', 'mxchat'),
274		- UPLOAD_ERR_CANT_WRITE => __('Server failed to write file to disk.', 'mxchat'),
275		- );
276		- $error_msg = isset($error_messages[$error_code]) ? $error_messages[$error_code] : __('Unknown upload error.', 'mxchat');
277		- set_transient('mxchat_admin_notice_error', $error_msg, 30);
278		- wp_safe_redirect(esc_url($redirect_url));
279		- exit;
280		- }
281		-
282		- $file = $_FILES['pdf_file'];
283		-
284		- // Validate MIME type
285		- $finfo = finfo_open(FILEINFO_MIME_TYPE);
286		- $mime_type = finfo_file($finfo, $file['tmp_name']);
287		- finfo_close($finfo);
288		-
289		- if ($mime_type !== 'application/pdf') {
290		- set_transient('mxchat_admin_notice_error',
291		- esc_html__('Invalid file type. Only PDF files are accepted.', 'mxchat'),
292		- 30
293		- );
294		- wp_safe_redirect(esc_url($redirect_url));
295		- exit;
296		- }
297		-
298		- // Validate extension
299		- $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
300		- if ($ext !== 'pdf') {
301		- set_transient('mxchat_admin_notice_error',
302		- esc_html__('Invalid file extension. Only .pdf files are accepted.', 'mxchat'),
303		- 30
304		- );
305		- wp_safe_redirect(esc_url($redirect_url));
306		- exit;
307		- }
308		-
309		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
310		- $original_filename = sanitize_file_name($file['name']);
311		-
312		- $upload_dir = wp_upload_dir();
313		- if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
314		- set_transient('mxchat_admin_notice_error',
315		- esc_html__('WordPress upload directory is not writable.', 'mxchat'),
316		- 30
317		- );
318		- wp_safe_redirect(esc_url($redirect_url));
319		- exit;
320		- }
321		-
322		- $pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
323		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
324		-
325		- if (!wp_mkdir_p(dirname($pdf_path))) {
326		- set_transient('mxchat_admin_notice_error',
327		- esc_html__('Failed to create upload directory.', 'mxchat'),
328		- 30
329		- );
330		- wp_safe_redirect(esc_url($redirect_url));
331		- exit;
332		- }
333		-
334		- // Move uploaded file
335		- if (!move_uploaded_file($file['tmp_name'], $pdf_path)) {
336		- set_transient('mxchat_admin_notice_error',
337		- esc_html__('Failed to save uploaded PDF file.', 'mxchat'),
338		- 30
339		- );
340		- wp_safe_redirect(esc_url($redirect_url));
341		- exit;
342		- }
343		-
344	526	try {
345		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
346		-
347		- if ($total_pages === false || $total_pages < 1) {
348		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
	527	+ if (!file_exists($pdf_path)) {
	528	+ throw new Exception(sprintf('PDF file not found at path: %s', esc_html($pdf_path)));
349	529	}
350	530
351		- // Use original filename as the source identifier
352		- $source_label = 'upload://' . $original_filename;
353		-
354		- $queue_id = 'pdf_' . md5($source_label . time());
355		-
356		- $pages = array();
357		- for ($i = 1; $i <= $total_pages; $i++) {
358		- $pages[] = array(
359		- 'pdf_path' => $pdf_path,
360		- 'pdf_url' => $source_label,
361		- 'page_number' => $i,
362		- 'total_pages' => $total_pages,
363		- );
364		- }
365		-
366		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
367		-
368		- if ($queued_count === 0) {
369		- wp_delete_file($pdf_path);
370		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
371		- }
372		-
373		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $source_label);
374		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
375		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
376		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
377		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
378		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
379		-
380		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
381		- set_transient('mxchat_last_pdf_url', $source_label, DAY_IN_SECONDS);
382		-
383		- set_transient('mxchat_admin_notice_success',
384		- sprintf(
385		- esc_html__('PDF "%s" (%d pages) queued for processing. Processing will start automatically.', 'mxchat'),
386		- esc_html($original_filename),
387		- $total_pages
388		- ),
389		- 30
390		- );
391		-
392		- } catch (Exception $e) {
393		- if (file_exists($pdf_path)) {
394		- wp_delete_file($pdf_path);
395		- }
396		- set_transient('mxchat_admin_notice_error',
397		- esc_html__('Failed to process uploaded PDF: ', 'mxchat') . esc_html($e->getMessage()),
398		- 30
399		- );
400		- }
401		-
402		- wp_safe_redirect(esc_url($redirect_url));
403		- exit;
404		-}
405		-
406		-/**
407		- * Validate PDF and count pages with multiple parser attempts
408		- */
409		-private function mxchat_validate_and_count_pdf_pages($pdf_path) {
410		- // Method 1: Try with Smalot PDF Parser (your current method)
411		- try {
412		- mxchat_load_pdf_parser();
413	531	$parser = new \Smalot\PdfParser\Parser();
414	532	$pdf = $parser->parseFile($pdf_path);
415	533	$pages = $pdf->getPages();
416		- $page_count = count($pages);
417		-
418		- if ($page_count > 0) {
419		- //error_log('PDF parsed successfully with Smalot parser: ' . $page_count . ' pages');
420		- return $page_count;
421		- }
422		- } catch (Exception $e) {
423		- //error_log('Smalot PDF parser failed: ' . $e->getMessage());
424		- }
425	534
426		- // Method 2: Try with pdfinfo command (if available)
427		- if (function_exists('shell_exec') && !$this->mxchat_is_shell_disabled()) {
428		- try {
429		- $command = 'pdfinfo ' . escapeshellarg($pdf_path) . ' 2>&1';
430		- $output = shell_exec($command);
431		-
432		- if ($output && preg_match('/Pages:\s*(\d+)/', $output, $matches)) {
433		- $page_count = intval($matches[1]);
434		- if ($page_count > 0) {
435		- //error_log('PDF parsed successfully with pdfinfo: ' . $page_count . ' pages');
436		- return $page_count;
437		- }
438		- }
439		- } catch (Exception $e) {
440		- //error_log('pdfinfo command failed: ' . $e->getMessage());
	535	+ // Get current progress
	536	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	537	+ $status = get_transient($status_key);
	538	+
	539	+ if (!$status || !is_array($status)) {
	540	+ throw new Exception('Invalid status data retrieved from transient');
441	541	}
442		- }
443	542
444		- // Method 3: Try to repair PDF and parse again
445		- try {
446		- $repaired_path = $this->mxchat_attempt_pdf_repair($pdf_path);
447		- if ($repaired_path && $repaired_path !== $pdf_path) {
448		- mxchat_load_pdf_parser();
449		- $parser = new \Smalot\PdfParser\Parser();
450		- $pdf = $parser->parseFile($repaired_path);
451		- $pages = $pdf->getPages();
452		- $page_count = count($pages);
453		-
454		- if ($page_count > 0) {
455		- // Replace original with repaired version
456		- copy($repaired_path, $pdf_path);
457		- unlink($repaired_path);
458		- //error_log('PDF repaired and parsed successfully: ' . $page_count . ' pages');
459		- return $page_count;
460		- }
461		-
462		- // Clean up repaired file if it didn't work
463		- unlink($repaired_path);
	543	+ // Initialize failed pages list if it doesn't exist
	544	+ if (!isset($status['failed_pages_list']) || !is_array($status['failed_pages_list'])) {
	545	+ $status['failed_pages_list'] = [];
464	546	}
465		- } catch (Exception $e) {
466		- //error_log('PDF repair attempt failed: ' . $e->getMessage());
467		- }
468	547
469		- // Method 4: Manual PDF structure analysis (basic page count)
470		- try {
471		- $page_count = $this->mxchat_manual_pdf_page_count($pdf_path);
472		- if ($page_count > 0) {
473		- //error_log('PDF page count determined manually: ' . $page_count . ' pages');
474		- return $page_count;
	548	+ $start_page = absint($status['processed_pages']);
	549	+ $end_page = min($start_page + $batch_size, $total_pages);
	550	+ $options = get_option('mxchat_options');
	551	+
	552	+ if (empty($options['api_key'])) {
	553	+ throw new Exception('API key is missing or invalid');
475	554	}
476		- } catch (Exception $e) {
477		- //error_log('Manual PDF analysis failed: ' . $e->getMessage());
478		- }
479	555
480		- //error_log('All PDF parsing methods failed for: ' . $pdf_path);
481		- return false;
482		-}
	556	+ $successful_pages = 0;
	557	+ $failed_pages = 0;
483	558
484		-/**
485		- * Check if shell_exec is disabled
486		- */
487		-private function mxchat_is_shell_disabled() {
488		- $disabled = explode(',', ini_get('disable_functions'));
489		- return in_array('shell_exec', $disabled);
490		-}
	559	+ for ($i = $start_page; $i < $end_page; $i++) {
	560	+ $page_number = $i + 1;
	561	+ $max_retries = 3;
	562	+ $retry_count = 0;
	563	+ $page_processed = false;
	564	+ $last_error = '';
491	565
492		-/**
493		- * Attempt to repair PDF using basic methods
494		- */
495		-private function mxchat_attempt_pdf_repair($pdf_path) {
496		- try {
497		- $content = file_get_contents($pdf_path);
498		- if (!$content) {
499		- return false;
500		- }
	566	+ while (!$page_processed && $retry_count < $max_retries) {
	567	+ try {
	568	+ $text = $pages[$i]->getText();
	569	+
	570	+ if (empty($text)) {
	571	+ throw new Exception("Empty text on page {$page_number}");
	572	+ }
	573	+
	574	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
501	575
502		- // Check if PDF starts with proper header
503		- if (substr($content, 0, 4) !== '%PDF') {
504		- // Try to find PDF header in the content
505		- $header_pos = strpos($content, '%PDF');
506		- if ($header_pos !== false && $header_pos < 1024) {
507		- // Remove junk before PDF header
508		- $content = substr($content, $header_pos);
509		- $repaired_path = $pdf_path . '.repaired';
510		- file_put_contents($repaired_path, $content);
511		- return $repaired_path;
512		- }
513		- }
	576	+ if (empty($sanitized_content)) {
	577	+ throw new Exception("No valid content after sanitization on page {$page_number}");
	578	+ }
514	579
515		- // Check for EOF marker
516		- $content = rtrim($content);
517		- if (!preg_match('/%%EOF\s*$/', $content)) {
518		- // Add EOF marker if missing
519		- $content .= "\n%%EOF";
520		- $repaired_path = $pdf_path . '.repaired';
521		- file_put_contents($repaired_path, $content);
522		- return $repaired_path;
523		- }
	580	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	581	+
	582	+ if (is_string($embedding_vector)) {
	583	+ throw new Exception("Embedding generation failed: " . $embedding_vector);
	584	+ }
	585	+
	586	+ if (!is_array($embedding_vector)) {
	587	+ throw new Exception("Embedding generation returned unexpected result type: " . gettype($embedding_vector));
	588	+ }
	589	+
	590	+ $metadata = array(
	591	+ 'document_type' => 'pdf',
	592	+ 'total_pages' => $total_pages,
	593	+ 'current_page' => $page_number,
	594	+ 'prev_page' => $i > 0 ? $i : null,
	595	+ 'next_page' => $i < ($total_pages - 1) ? $i + 2 : null,
	596	+ 'source_url' => $pdf_url
	597	+ );
524	598
525		- } catch (Exception $e) {
526		- //error_log('PDF repair error: ' . $e->getMessage());
527		- }
	599	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	600	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
528	601
529		- return false;
530		-}
	602	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $options['api_key']);
	603	+
	604	+ if (is_wp_error($db_result)) {
	605	+ throw new Exception("Database submission failed: " . $db_result->get_error_message());
	606	+ }
531	607
532		-/**
533		- * Manual PDF page counting by analyzing PDF structure
534		- */
535		-private function mxchat_manual_pdf_page_count($pdf_path) {
536		- try {
537		- $content = file_get_contents($pdf_path);
538		- if (!$content) {
539		- return 0;
540		- }
	608	+ // Success!
	609	+ $page_processed = true;
	610	+ $successful_pages++;
	611	+
	612	+ } catch (Exception $e) {
	613	+ $retry_count++;
	614	+ $last_error = $e->getMessage();
	615	+
	616	+ //error_log("PDF page {$page_number} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	617	+
	618	+ if ($retry_count < $max_retries) {
	619	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	620	+ sleep(pow(2, $retry_count - 1));
	621	+ }
	622	+ }
	623	+ }
541	624
542		- // Method 1: Count /Type /Page objects
543		- $page_count = preg_match_all('/\/Type\s*\/Page[^s]/', $content);
544		- if ($page_count > 0) {
545		- return $page_count;
546		- }
	625	+ // If page still not processed after all retries, mark as failed
	626	+ if (!$page_processed) {
	627	+ $failed_pages++;
	628	+ $status['failed_pages_list'][] = [
	629	+ 'page' => $page_number,
	630	+ 'error' => $last_error,
	631	+ 'time' => time(),
	632	+ 'retries' => $max_retries
	633	+ ];
	634	+
	635	+ // Limit failed pages list to prevent memory issues
	636	+ if (count($status['failed_pages_list']) > 50) {
	637	+ $status['failed_pages_list'] = array_slice($status['failed_pages_list'], -50);
	638	+ }
	639	+
	640	+ //error_log("PDF page {$page_number} permanently failed after {$max_retries} attempts: " . $last_error);
	641	+ }
547	642
548		- // Method 2: Look for /Count in pages object
549		- if (preg_match('/\/Type\s*\/Pages.*?\/Count\s+(\d+)/', $content, $matches)) {
550		- return intval($matches[1]);
	643	+ // Update progress
	644	+ $status['processed_pages'] = absint($page_number);
	645	+ $status['last_update'] = time();
	646	+ $status['failed_pages'] = absint($status['failed_pages'] ?? 0) + ($page_processed ? 0 : 1);
	647	+
	648	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
551	649	}
552	650
553		- // Method 3: Count page references
554		- $page_count = preg_match_all('/\d+\s+0\s+obj\s*<<[^>]*\/Type\s*\/Page/', $content);
555		- if ($page_count > 0) {
556		- return $page_count;
557		- }
558		-
559		- } catch (Exception $e) {
560		- //error_log('Manual PDF analysis error: ' . $e->getMessage());
561		- }
562		-
563		- return 0;
564		-}
565		-
566		-
567		-public function mxchat_save_inline_prompt() {
568		- // DEBUG: Log what we're receiving
569		- //error_log('=== MXCHAT DEBUG ===');
570		- //error_log('POST data: ' . print_r($_POST, true));
571		- //error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
572		-
573		- // Check for nonce security
574		- check_ajax_referer('mxchat_save_inline_nonce', '_ajax_nonce');
575		-
576		- // If we get here, nonce passed
577		- //error_log('Nonce verification PASSED');
578		-
579		- // Verify permissions
580		- if (!current_user_can('manage_options')) {
581		- wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
582		- return;
583		- }
584		-
585		- global $wpdb;
586		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
587		-
588		- // Validate and sanitize input data
589		- $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
590		- $article_content = isset($_POST['article_content']) ? wp_kses_post(wp_unslash($_POST['article_content'])) : '';
591		- $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
592		-
593		- if ($prompt_id > 0 && !empty($article_content)) {
594		- // Re-generate the embedding vector for the updated content
595		- $embedding_vector = $this->mxchat_generate_embedding($article_content);
596		- if (is_array($embedding_vector)) {
597		- // Serialize the embedding vector before storing it
598		- $embedding_vector_serialized = serialize($embedding_vector);
599		- // Update the prompt in the database
600		- $updated = $wpdb->update(
601		- $table_name,
602		- array(
603		- 'article_content' => $article_content,
604		- 'embedding_vector' => $embedding_vector_serialized,
605		- 'source_url' => $article_url,
606		- ),
607		- array('id' => $prompt_id),
608		- array('%s', '%s', '%s'),
609		- array('%d')
610		- );
611		- if ($updated !== false) {
612		- wp_send_json_success();
613		- } else {
614		- MxChat_Admin::mxchat_log_debug('knowledge_error', 'Database update failed when saving knowledge entry');
615		- wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
616		- }
	651	+ // Schedule next batch if needed
	652	+ if ($end_page < $total_pages) {
	653	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_pdf_pages', array(
	654	+ 'pdf_path' => $pdf_path,
	655	+ 'pdf_url' => $pdf_url,
	656	+ 'total_pages' => $total_pages,
	657	+ 'batch_size' => $batch_size,
	658	+ 'batch_pause' => $batch_pause
	659	+ ));
617	660	} else {
618		- MxChat_Admin::mxchat_log_debug('embedding_error', 'Embedding generation failed for knowledge entry');
619		- wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
620		- }
621		- } else {
622		- wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
623		- }
624		-}
625		-
626		-
627		-/**
628		- * AJAX: Get full content for editing — reassembles chunks if needed.
629		- * Works for both WordPress DB and Pinecone entries.
630		- */
631		-public function ajax_mxchat_get_entry_content() {
632		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
633		-
634		- if ( ! current_user_can('manage_options') ) {
635		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
636		- }
637		-
638		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
639		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
640		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
641		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
642		-
643		- if ( $data_source === 'pinecone' ) {
644		- // Pinecone: fetch vectors by source_url, reassemble chunks
645		- $content = $this->get_pinecone_entry_content( $source_url, $entry_id, $bot_id );
646		- } else {
647		- // WordPress DB
648		- $content = $this->get_wordpress_entry_content( $source_url, $entry_id );
649		- }
650		-
651		- if ( is_wp_error( $content ) ) {
652		- wp_send_json_error( array( 'message' => $content->get_error_message() ) );
653		- }
654		-
655		- wp_send_json_success( $content );
656		-}
657		-
658		-/**
659		- * Get content from WordPress DB — reassembles chunks by source_url.
660		- */
661		-private function get_wordpress_entry_content( $source_url, $entry_id ) {
662		- global $wpdb;
663		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
664		-
665		- // If we have a source_url, check for chunks
666		- if ( ! empty( $source_url ) && strpos( $source_url, 'mxchat://' ) !== 0 ) {
667		- $rows = $wpdb->get_results( $wpdb->prepare(
668		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
669		- $source_url
670		- ) );
671		-
672		- if ( $rows && count( $rows ) > 1 ) {
673		- // Multiple rows = chunked. Reassemble.
674		- $chunks = array();
675		- foreach ( $rows as $row ) {
676		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
677		- $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
678		- $chunks[ $index ] = $parsed['text'];
	661	+ // Processing complete
	662	+ $status['status'] = 'complete';
	663	+ $status['processed_pages'] = $total_pages;
	664	+
	665	+ // Add completion summary
	666	+ $status['completion_summary'] = [
	667	+ 'total_pages' => $total_pages,
	668	+ 'successful_pages' => $total_pages - absint($status['failed_pages'] ?? 0),
	669	+ 'failed_pages' => absint($status['failed_pages'] ?? 0),
	670	+ 'completion_time' => current_time('mysql')
	671	+ ];
	672	+
	673	+ // Save the completed status (don't delete it - let user dismiss manually)
	674	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	675	+
	676	+ // Clean up the temporary PDF file
	677	+ if (file_exists($pdf_path)) {
	678	+ wp_delete_file($pdf_path);
679	679	}
680		- ksort( $chunks );
681		- return array(
682		- 'content' => implode( "\n\n", $chunks ),
683		- 'source_url' => $source_url,
684		- 'is_chunked' => true,
685		- 'chunk_count' => count( $chunks ),
686		- 'content_type' => $rows[0]->content_type,
687		- );
688		- } elseif ( $rows && count( $rows ) === 1 ) {
689		- $parsed = MxChat_Chunker::parse_stored_chunk( $rows[0]->article_content );
690		- return array(
691		- 'content' => $parsed['text'],
692		- 'source_url' => $source_url,
693		- 'entry_id' => $rows[0]->id,
694		- 'is_chunked' => false,
695		- 'content_type' => $rows[0]->content_type,
696		- );
	680	+
	681	+ // DON'T delete the status transients here - let user dismiss manually
697	682	}
698		- }
699	683
700		- // Fallback: fetch by ID
701		- if ( $entry_id > 0 ) {
702		- $row = $wpdb->get_row( $wpdb->prepare(
703		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
704		- $entry_id
705		- ) );
706		- if ( $row ) {
707		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
708		- return array(
709		- 'content' => $parsed['text'],
710		- 'source_url' => $row->source_url,
711		- 'entry_id' => $row->id,
712		- 'is_chunked' => false,
713		- 'content_type' => $row->content_type,
	684	+ } catch (\Exception $e) {
	685	+ //error_log(sprintf('[MXCHAT-PDF] Error processing PDF: %s', $e->getMessage()));
	686	+
	687	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	688	+ $status = get_transient($status_key);
	689	+
	690	+ if (!$status || !is_array($status)) {
	691	+ $status = array(
	692	+ 'total_pages' => $total_pages,
	693	+ 'processed_pages' => 0,
	694	+ 'status' => 'error',
	695	+ 'error' => sanitize_text_field($e->getMessage()),
	696	+ 'last_update' => time()
714	697	);
	698	+ } else {
	699	+ $status['status'] = 'error';
	700	+ $status['error'] = sanitize_text_field($e->getMessage());
	701	+ $status['last_update'] = time();
715	702	}
716		- }
717		-
718		- return new WP_Error( 'not_found', 'Entry not found.' );
719		-}
720		-
721		-/**
722		- * Get content from Pinecone — fetches vectors by source_url, reassembles chunks.
723		- */
724		-private function get_pinecone_entry_content( $source_url, $entry_id, $bot_id ) {
725		- if ( ! class_exists('MxChat_Pinecone_Manager') ) {
726		- return new WP_Error( 'pinecone_unavailable', 'Pinecone manager not available.' );
727		- }
728		-
729		- // Get Pinecone config
730		- if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
731		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
732		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
733		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
734		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
735		- } else {
736		- $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
737		- $api_key = $bot_config['api_key'] ?? '';
738		- $host = $bot_config['host'] ?? '';
739		- $namespace = $bot_config['namespace'] ?? '';
740		- }
741		-
742		- if ( empty($host) || empty($api_key) ) {
743		- return new WP_Error( 'pinecone_config', 'Pinecone not configured.' );
744		- }
745		-
746		- // List vectors with the source_url prefix
747		- $base_id = md5( $source_url );
748		- $vector_ids = array( $base_id );
749		-
750		- // Find chunk vectors
751		- $list_url = "https://{$host}/vectors/list";
752		- $list_body = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
753		- if ( ! empty($namespace) ) {
754		- $list_body['namespace'] = $namespace;
755		- }
756		-
757		- $list_resp = wp_remote_post( $list_url, array(
758		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
759		- 'body' => wp_json_encode( $list_body ),
760		- 'timeout' => 15,
761		- ) );
762		-
763		- if ( ! is_wp_error($list_resp) ) {
764		- $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
765		- if ( ! empty($list_data['vectors']) ) {
766		- foreach ( $list_data['vectors'] as $v ) {
767		- $vector_ids[] = $v['id'];
768		- }
	703	+
	704	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	705	+
	706	+ if (file_exists($pdf_path)) {
	707	+ wp_delete_file($pdf_path);
769	708	}
770	709	}
771		-
772		- // Fetch vectors with metadata
773		- $fetch_url = "https://{$host}/vectors/fetch";
774		- $fetch_body = array( 'ids' => $vector_ids );
775		- if ( ! empty($namespace) ) {
776		- $fetch_body['namespace'] = $namespace;
777		- }
778		-
779		- $fetch_resp = wp_remote_post( $fetch_url, array(
780		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
781		- 'body' => wp_json_encode( $fetch_body ),
782		- 'timeout' => 15,
783		- ) );
784		-
785		- if ( is_wp_error($fetch_resp) ) {
786		- return new WP_Error( 'pinecone_fetch', 'Failed to fetch from Pinecone.' );
787		- }
788		-
789		- $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
790		- $vectors = $fetch_data['vectors'] ?? array();
791		-
792		- if ( empty($vectors) ) {
793		- return new WP_Error( 'not_found', 'Entry not found in Pinecone.' );
794		- }
795		-
796		- // Reassemble chunks
797		- $chunks = array();
798		- $content_type = 'content';
799		- foreach ( $vectors as $vid => $vector ) {
800		- $meta = $vector['metadata'] ?? array();
801		- $text = $meta['text'] ?? '';
802		- $index = $meta['chunk_index'] ?? 0;
803		- $content_type = $meta['type'] ?? 'content';
804		- $chunks[ intval($index) ] = $text;
805		- }
806		- ksort( $chunks );
807		-
808		- return array(
809		- 'content' => implode( "\n\n", $chunks ),
810		- 'source_url' => $source_url,
811		- 'is_chunked' => count($chunks) > 1,
812		- 'chunk_count' => count($chunks),
813		- 'content_type' => $content_type,
814		- );
815	710	}
816	711
817		-/**
818		- * AJAX: Save edited content — re-chunks and re-embeds as needed.
819		- * Works for both WordPress DB and Pinecone entries.
820		- */
821		-public function ajax_mxchat_save_entry_content() {
822		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
	712	+ public function mxchat_save_inline_prompt() {
	713	+ // Check for nonce security
	714	+ check_ajax_referer('mxchat_save_inline_nonce');
823	715
824		- if ( ! current_user_can('manage_options') ) {
825		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
826		- }
	716	+ // Verify permissions
	717	+ if (!current_user_can('manage_options')) {
	718	+ wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
	719	+ return;
	720	+ }
827	721
828		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
829		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
830		- $content = isset($_POST['content']) ? wp_kses_post( wp_unslash($_POST['content']) ) : '';
831		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
832		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
833		- $content_type = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : 'content';
	722	+ global $wpdb;
	723	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
834	724
835		- if ( empty($content) ) {
836		- wp_send_json_error( array( 'message' => 'Content cannot be empty.' ) );
837		- }
	725	+ // Validate and sanitize input data
	726	+ $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
	727	+ $article_content = isset($_POST['article_content']) ? sanitize_textarea_field($_POST['article_content']) : '';
	728	+ $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
838	729
839		- // Get the embedding API key
840		- $options = get_option('mxchat_options', array());
841		- $api_key = '';
	730	+ if ($prompt_id > 0 && !empty($article_content)) {
	731	+ // Re-generate the embedding vector for the updated content
	732	+ $embedding_vector = $this->mxchat_generate_embedding($article_content);
842	733
843		- if ( $bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager') ) {
844		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
845		- $api_key = $bot_options['api_key'] ?? '';
846		- }
847		- if ( empty($api_key) ) {
848		- $api_key = $options['api_key'] ?? '';
849		- }
	734	+ if (is_array($embedding_vector)) {
	735	+ // Serialize the embedding vector before storing it
	736	+ $embedding_vector_serialized = serialize($embedding_vector);
850	737
851		- global $wpdb;
852		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
	738	+ // Update the prompt in the database
	739	+ $updated = $wpdb->update(
	740	+ $table_name,
	741	+ array(
	742	+ 'article_content' => $article_content,
	743	+ 'embedding_vector' => $embedding_vector_serialized,
	744	+ 'source_url' => $article_url,
	745	+ ),
	746	+ array('id' => $prompt_id),
	747	+ array('%s', '%s', '%s'),
	748	+ array('%d')
	749	+ );
853	750
854		- // If source_url is empty but we have an entry_id, look it up
855		- if ( empty($source_url) && $entry_id > 0 && $data_source === 'wordpress' ) {
856		- $row = $wpdb->get_row( $wpdb->prepare( "SELECT source_url FROM {$table} WHERE id = %d", $entry_id ) );
857		- if ( $row && ! empty($row->source_url) ) {
858		- $source_url = $row->source_url;
859		- }
860		- }
	751	+ if ($updated !== false) {
	752	+ wp_send_json_success();
	753	+ } else {
	754	+ wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
	755	+ }
	756	+ } else {
	757	+ wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
	758	+ }
	759	+ } else {
	760	+ wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
	761	+ }
	762	+ }
861	763
862		- // For manual entries (no source_url or mxchat:// prefix), delete the old entry by ID first
863		- // so submit_content_to_db creates a replacement instead of a duplicate
864		- // Also treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
865		- $is_legacy_manual = !empty($source_url) && strpos($source_url, 'mxchat.ai') !== false && strpos($source_url, 'mxchat://') !== 0;
866		- if ( $entry_id > 0 && (empty($source_url) || strpos($source_url, 'mxchat://') === 0 || $is_legacy_manual) ) {
867		- $wpdb->delete( $table, array( 'id' => $entry_id ), array( '%d' ) );
868		- // Clear legacy URL so submit_content_to_db generates a unique mxchat:// identifier
869		- // instead of reusing the shared URL (which would mass-delete other entries with the same URL)
870		- if ( $is_legacy_manual ) {
871		- $source_url = '';
872		- }
873		- }
874	764
875		- // Use the existing submit_content_to_db which handles chunking, Pinecone, and WP DB
876		- $vector_id = ! empty($source_url) ? md5($source_url) : md5('mxchat_manual_' . $entry_id);
877		-
878		- // submit_content_to_db already handles: delete old chunks → re-chunk → re-embed → store
879		- $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, $vector_id, $bot_id, $content_type );
880		-
881		- if ( is_wp_error($result) ) {
882		- wp_send_json_error( array( 'message' => $result->get_error_message() ) );
883		- }
884		-
885		- wp_send_json_success( array( 'message' => 'Content saved and re-embedded successfully.' ) );
886		-}
887		-
888	765	public function mxchat_get_pdf_processing_status($pdf_url) {
889	766	$pdf_url = esc_url_raw($pdf_url);
890	767	$status = get_transient(sanitize_key('mxchat_pdf_status_' . md5($pdf_url)));
891	768
		@@ -928,18 +805,24 @@
928	805	}
929	806
930	807
931	808	public function mxchat_handle_sitemap_submission() {
	809	+ // Start logging the submission process
	810	+ //error_log('[MXCHAT-URL] ===== Starting URL submission process =====');
	811	+
932	812	// Check if the form was submitted and verify permissions
933	813	if (!isset($_POST['submit_sitemap']) || !current_user_can('manage_options')) {
	814	+ //error_log('[MXCHAT-URL] Error: Unauthorized access or form not submitted properly');
934	815	wp_die(esc_html__('Unauthorized access', 'mxchat'));
935	816	}
936	817
937	818	// Verify nonce
	819	+ //error_log('[MXCHAT-URL] Verifying nonce');
938	820	check_admin_referer('mxchat_submit_sitemap_action', 'mxchat_submit_sitemap_nonce');
939	821
940	822	// Validate URL
941	823	if (!isset($_POST['sitemap_url']) || empty($_POST['sitemap_url'])) {
	824	+ //error_log('[MXCHAT-URL] Error: Empty or missing URL');
942	825	set_transient('mxchat_admin_notice_error',
943	826	esc_html__('Please provide a valid URL.', 'mxchat'),
944	827	30
945	828	);
		@@ -947,28 +830,12 @@
947	830	exit;
948	831	}
949	832
950	833	$submitted_url = esc_url_raw($_POST['sitemap_url']);
	834	+ //error_log('[MXCHAT-URL] Processing URL: ' . $submitted_url);
951	835
952		- // Convert Google Drive sharing URLs to direct download URLs
953		- if ( strpos($submitted_url, 'drive.google.com') !== false ) {
954		- $file_id = '';
955		- if ( preg_match('/[?&]id=([a-zA-Z0-9_-]+)/', $submitted_url, $m) ) {
956		- $file_id = $m[1];
957		- } elseif ( preg_match('#/file/d/([a-zA-Z0-9_-]+)#', $submitted_url, $m) ) {
958		- $file_id = $m[1];
959		- }
960		- if ( ! empty($file_id) ) {
961		- $submitted_url = 'https://drive.google.com/uc?export=download&id=' . $file_id;
962		- }
963		- }
964		-
965		- // Get bot_id from form submission
966		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
967		-
968		- // Get bot-specific options and validate API key
969		- $bot_options = $this->get_bot_options($bot_id);
970		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	836	+ // Validate API key first
	837	+ $options = get_option('mxchat_options');
971	838	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
972	839
973	840	if (strpos($selected_model, 'voyage') === 0) {
974	841	$api_key = $options['voyage_api_key'] ?? '';
		@@ -985,26 +852,21 @@
985	852	$error_message = sprintf(
986	853	esc_html__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
987	854	$provider_name
988	855	);
	856	+ //error_log('[MXCHAT-URL] Error: ' . $error_message);
989	857	set_transient('mxchat_admin_notice_error', $error_message, 30);
	858	+ //error_log('[MXCHAT-URL] Set error transient: ' . $error_message);
990	859	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
991	860	exit;
992	861	}
993	862
994		- // Fetch URL — use browser-like headers so servers with bot protection don't block us
995		- $response = wp_remote_get($submitted_url, array(
996		- 'timeout' => 30,
997		- 'sslverify' => false,
998		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
999		- 'headers' => array(
1000		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1001		- 'Accept-Language' => 'en-US,en;q=0.9',
1002		- ),
1003		- ));
	863	+ //error_log('[MXCHAT-URL] Fetching URL content');
	864	+ $response = wp_remote_get($submitted_url, array('timeout' => 30));
1004	865
1005	866	if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
1006	867	$error_message = is_wp_error($response) ? $response->get_error_message() : 'HTTP Status: ' . wp_remote_retrieve_response_code($response);
	868	+ //error_log('[MXCHAT-URL] Error fetching URL: ' . $error_message);
1007	869	set_transient('mxchat_admin_notice_error',
1008	870	sprintf(
1009	871	esc_html__('Failed to fetch the URL: %s', 'mxchat'),
1010	872	esc_html($error_message)
		@@ -1015,11 +877,13 @@
1015	877	exit;
1016	878	}
1017	879
1018	880	$content_type = wp_remote_retrieve_header($response, 'content-type');
	881	+ //error_log('[MXCHAT-URL] Content type: ' . $content_type);
1019	882	$body_content = wp_remote_retrieve_body($response);
1020	883
1021	884	if (empty($body_content)) {
	885	+ //error_log('[MXCHAT-URL] Error: Empty response body');
1022	886	set_transient('mxchat_admin_notice_error',
1023	887	esc_html__('Empty response received from URL.', 'mxchat'),
1024	888	30
1025	889	);
		@@ -1025,21 +889,29 @@
1025	889	);
1026	890	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1027	891	exit;
1028	892	}
	893	+ //error_log('[MXCHAT-URL] Retrieved body content length: ' . strlen($body_content) . ' bytes');
1029	894
1030	895	// Handle PDF URL
1031	896	if ($this->mxchat_is_pdf_url($submitted_url, $response)) {
1032		- $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response, $bot_id);
	897	+ //error_log('[MXCHAT-URL] Detected PDF URL, handling PDF for knowledge base');
	898	+ $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response);
	899	+ //error_log('[MXCHAT-URL] PDF handling result: ' . $result);
1033	900
1034		- if ($result === 'queued') {
1035		- set_transient('mxchat_admin_notice_success',
1036		- esc_html__('PDF queued for processing. Processing will start automatically.', 'mxchat'),
	901	+ if ($result === 'scheduled') {
	902	+ set_transient(
	903	+ 'mxchat_last_pdf_url',
	904	+ sanitize_text_field($submitted_url),
	905	+ DAY_IN_SECONDS
	906	+ );
	907	+ set_transient('mxchat_admin_notice_info',
	908	+ esc_html__('PDF processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1037	909	30
1038	910	);
1039	911	} else {
1040	912	set_transient('mxchat_admin_notice_error',
1041		- esc_html__('Failed to queue PDF processing: ', 'mxchat') . esc_html($result),
	913	+ esc_html__('Failed to start PDF processing: ', 'mxchat') . esc_html($result),
1042	914	30
1043	915	);
1044	916	}
1045	917
		@@ -1048,8 +920,9 @@
1048	920	}
1049	921
1050	922	// Handle Sitemap XML
1051	923	if (strpos($content_type, 'xml') !== false || strpos($body_content, '<urlset') !== false) {
	924	+ //error_log('[MXCHAT-URL] Detected XML content, processing as sitemap');
1052	925	libxml_use_internal_errors(true);
1053	926	$xml = simplexml_load_string($body_content);
1054	927	$xml_errors = libxml_get_errors();
1055	928	libxml_clear_errors();
		@@ -1054,8 +927,15 @@
1054	927	$xml_errors = libxml_get_errors();
1055	928	libxml_clear_errors();
1056	929
1057	930	if ($xml === false || !empty($xml_errors)) {
	931	+ //error_log('[MXCHAT-URL] Error: Invalid XML format');
	932	+ if (!empty($xml_errors)) {
	933	+ foreach ($xml_errors as $error) {
	934	+ //error_log('[MXCHAT-URL] XML Error: ' . $error->message);
	935	+ }
	936	+ }
	937	+
1058	938	set_transient('mxchat_admin_notice_error',
1059	939	esc_html__('Invalid sitemap XML. Please provide a valid sitemap.', 'mxchat'),
1060	940	30
1061	941	);
		@@ -1062,18 +942,27 @@
1062	942	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1063	943	exit;
1064	944	}
1065	945
1066		- $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url, $bot_id);
	946	+ //error_log('[MXCHAT-URL] Valid XML found, handling sitemap for knowledge base');
	947	+ $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url);
	948	+ //error_log('[MXCHAT-URL] Sitemap handling result: ' . $result);
1067	949
1068		- if ($result === 'queued') {
1069		- set_transient('mxchat_admin_notice_success',
1070		- esc_html__('Sitemap queued for processing. Processing will start automatically.', 'mxchat'),
	950	+ if ($result === 'scheduled') {
	951	+ set_transient(
	952	+ 'mxchat_last_sitemap_url',
	953	+ sanitize_text_field($submitted_url),
	954	+ DAY_IN_SECONDS
	955	+ );
	956	+ set_transient('mxchat_admin_notice_info',
	957	+ esc_html__('Sitemap processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1071	958	30
1072	959	);
1073	960	} else {
	961	+ // Return to the admin page without a redirect for better error display
	962	+ // The error is already stored in the sitemap status transient
1074	963	set_transient('mxchat_admin_notice_error',
1075		- esc_html__('Failed to queue sitemap processing. Please check the status below for details.', 'mxchat'),
	964	+ esc_html__('Failed to start sitemap processing. Please check the status below for details.', 'mxchat'),
1076	965	30
1077	966	);
1078	967	}
1079	968
		@@ -1080,50 +969,127 @@
1080	969	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1081	970	exit;
1082	971	}
1083	972
1084		- // Handle Regular URL (single page)
	973	+ // Handle Regular URL
	974	+ //error_log('[MXCHAT-URL] Processing as regular webpage');
1085	975	$page_content = $this->mxchat_extract_main_content($body_content);
	976	+ //error_log('[MXCHAT-URL] Extracted content length: ' . strlen($page_content) . ' bytes');
	977	+
1086	978	$sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	979	+ //error_log('[MXCHAT-URL] Sanitized content length: ' . strlen($sanitized_content) . ' bytes');
1087	980
1088		- //error_log('[MXCHAT-URL-DEBUG] URL: ' . $submitted_url);
1089		- //error_log('[MXCHAT-URL-DEBUG] Extracted page_content length: ' . strlen($page_content));
1090		- //error_log('[MXCHAT-URL-DEBUG] Sanitized content length: ' . strlen($sanitized_content));
1091		- //error_log('[MXCHAT-URL-DEBUG] Content preview: ' . substr($sanitized_content, 0, 500));
1092		-
1093	981	if (empty($sanitized_content)) {
	982	+ //error_log('[MXCHAT-URL] Error: No valid content after sanitization');
	983	+
	984	+ // Set both transients - the error notice and the URL status
1094	985	set_transient('mxchat_admin_notice_error',
1095	986	esc_html__('No valid content found on the provided URL.', 'mxchat'),
1096	987	30
1097	988	);
	989	+
	990	+ // Set URL status transient
	991	+ set_transient('mxchat_single_url_status', [
	992	+ 'url' => $submitted_url,
	993	+ 'timestamp' => current_time('mysql'),
	994	+ 'status' => 'failed',
	995	+ 'error' => esc_html__('No valid content found on the provided URL.', 'mxchat')
	996	+ ], DAY_IN_SECONDS);
	997	+
1098	998	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1099	999	exit;
1100	1000	}
1101	1001
1102		- // For single URLs, process immediately using submit_content_to_db
1103		- // This handles chunking automatically for large content
1104		- $db_result = MxChat_Utils::submit_content_to_db(
1105		- $sanitized_content,
1106		- $submitted_url,
1107		- $api_key,
1108		- null,
1109		- $bot_id,
1110		- 'url' // content_type
1111		- );
	1002	+ //error_log('[MXCHAT-URL] Generating embedding for content');
	1003	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
1112	1004
1113		- if (is_wp_error($db_result)) {
1114		- $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1005	+ // Check if embedding_vector is a string (error message)
	1006	+ if (is_string($embedding_vector)) {
	1007	+ //error_log('[MXCHAT-URL] Error generating embedding: ' . $embedding_vector);
	1008	+ $error_message = esc_html__('Failed to generate embedding: ', 'mxchat') . esc_html($embedding_vector);
	1009	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1010	+
	1011	+ // Set both transients
1115	1012	set_transient('mxchat_admin_notice_error', $error_message, 30);
1116		- } else {
	1013	+
	1014	+ // Set URL status transient
	1015	+ set_transient('mxchat_single_url_status', [
	1016	+ 'url' => $submitted_url,
	1017	+ 'timestamp' => current_time('mysql'),
	1018	+ 'status' => 'failed',
	1019	+ 'error' => $error_message
	1020	+ ], DAY_IN_SECONDS);
	1021	+
	1022	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1023	+ exit;
	1024	+ }
	1025	+
	1026	+ if (is_array($embedding_vector)) {
	1027	+ //error_log('[MXCHAT-URL] Successfully generated embedding with ' . count($embedding_vector) . ' dimensions');
	1028	+
	1029	+ $db_result = MxChat_Utils::submit_content_to_db(
	1030	+ $sanitized_content,
	1031	+ $submitted_url,
	1032	+ $api_key
	1033	+ );
	1034	+
	1035	+ if (is_wp_error($db_result)) {
	1036	+ //error_log('[MXCHAT-URL] Error: Failed to store content in database: ' . $db_result->get_error_message());
	1037	+ $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1038	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1039	+
	1040	+ // Set both transients
	1041	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1042	+
	1043	+ // Set URL status transient
	1044	+ set_transient('mxchat_single_url_status', [
	1045	+ 'url' => $submitted_url,
	1046	+ 'timestamp' => current_time('mysql'),
	1047	+ 'status' => 'failed',
	1048	+ 'error' => $error_message
	1049	+ ], DAY_IN_SECONDS);
	1050	+
	1051	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1052	+ exit;
	1053	+ }
	1054	+
	1055	+ //error_log('[MXCHAT-URL] Successfully stored content in database');
1117	1056	$success_message = esc_html__('URL content successfully submitted!', 'mxchat');
	1057	+ //error_log('[MXCHAT-URL] Setting success transient: ' . $success_message);
	1058	+
	1059	+ // Set both transients
1118	1060	set_transient('mxchat_admin_notice_success', $success_message, 30);
	1061	+
	1062	+ // Set URL status transient with success
	1063	+ set_transient('mxchat_single_url_status', [
	1064	+ 'url' => $submitted_url,
	1065	+ 'timestamp' => current_time('mysql'),
	1066	+ 'status' => 'complete',
	1067	+ 'content_length' => strlen($sanitized_content),
	1068	+ 'embedding_dimensions' => count($embedding_vector)
	1069	+ ], DAY_IN_SECONDS);
	1070	+
	1071	+ } else {
	1072	+ //error_log('[MXCHAT-URL] Error: Failed to generate embedding. Unexpected result type: ' . gettype($embedding_vector));
	1073	+ $error_message = esc_html__('Failed to generate embedding: Unexpected result type. Please check your API key and try again.', 'mxchat');
	1074	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1075	+
	1076	+ // Set both transients
	1077	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1078	+
	1079	+ // Set URL status transient
	1080	+ set_transient('mxchat_single_url_status', [
	1081	+ 'url' => $submitted_url,
	1082	+ 'timestamp' => current_time('mysql'),
	1083	+ 'status' => 'failed',
	1084	+ 'error' => $error_message
	1085	+ ], DAY_IN_SECONDS);
1119	1086	}
1120	1087
	1088	+ //error_log('[MXCHAT-URL] ===== Completed URL submission process =====');
1121	1089	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1122	1090	exit;
1123	1091	}
1124		-
1125		-
1126	1092	public function mxchat_get_single_url_status() {
1127	1093	$status = get_transient('mxchat_single_url_status');
1128	1094	if (!$status) {
1129	1095	return null;
		@@ -1135,11 +1101,13 @@
1135	1101	}
1136	1102
1137	1103	return $status;
1138	1104	}
1139		-
1140		-public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url, $bot_id = 'default') {
	1105	+public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url) {
	1106	+ // Clear any single URL status when starting sitemap processing
	1107	+ delete_transient('mxchat_single_url_status');
1141	1108	if (!current_user_can('manage_options')) {
	1109	+ //error_log(esc_html__('Unauthorized sitemap processing attempt', 'mxchat'));
1142	1110	return false;
1143	1111	}
1144	1112
1145	1113	try {
		@@ -1148,96 +1116,331 @@
1148	1116	if (!$xml || !is_object($xml)) {
1149	1117	throw new Exception(__('Invalid XML object provided', 'mxchat'));
1150	1118	}
1151	1119
1152		- // Get bot-specific embedding API for validation
1153		- $bot_options = $this->get_bot_options($bot_id);
1154		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	1120	+ // Add embedding validation before processing
	1121	+ // Test embedding with a small sample text to verify API key is working
	1122	+ $test_result = $this->mxchat_generate_embedding("This is a test to verify the embedding API key is working.");
1155	1123
1156		- // Test the embedding API before processing
1157		- $test_phrase = "Test embedding generation for MxChat";
1158		- $test_result = $this->mxchat_generate_embedding($test_phrase, $bot_id);
1159		-
	1124	+ // Check if test_result is a string (error message) rather than an array (valid embedding)
1160	1125	if (is_string($test_result)) {
	1126	+ //error_log('[MXCHAT-URL] Embedding API validation failed: ' . $test_result);
	1127	+
	1128	+ // Store the error in the status transient so it can be displayed later
	1129	+ $status_data = array(
	1130	+ 'total_urls' => 0,
	1131	+ 'processed_urls' => 0,
	1132	+ 'status' => 'error',
	1133	+ 'error' => __('Embedding API validation failed: ', 'mxchat') . $test_result,
	1134	+ 'last_update' => time()
	1135	+ );
	1136	+
	1137	+ set_transient(
	1138	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1139	+ array_map('sanitize_text_field', $status_data),
	1140	+ DAY_IN_SECONDS
	1141	+ );
	1142	+
1161	1143	throw new Exception(__('Embedding API validation failed: ', 'mxchat') . $test_result);
1162	1144	}
1163	1145
	1146	+ // Make sure it's an array (valid embedding)
1164	1147	if (!is_array($test_result)) {
	1148	+ //error_log('[MXCHAT-URL] Embedding API returned unexpected result type: ' . gettype($test_result));
1165	1149	throw new Exception(__('Embedding API returned unexpected result type. Please check your configuration.', 'mxchat'));
1166	1150	}
1167	1151
1168		- // Extract URLs from sitemap
1169		- $urls = array();
	1152	+ $urls = [];
1170	1153	foreach ($xml->url as $url_element) {
1171	1154	$url = esc_url_raw((string)$url_element->loc);
1172	1155	if ($url) {
1173		- $urls[] = array('url' => $url);
	1156	+ $urls[] = $url;
1174	1157	}
1175	1158	}
1176	1159
1177		- $total_urls = count($urls);
	1160	+ $total_urls = absint(count($urls));
1178	1161
1179	1162	if ($total_urls < 1) {
1180	1163	throw new Exception(__('No valid URLs found in sitemap', 'mxchat'));
1181	1164	}
1182	1165
1183		- // Create unique queue ID
1184		- $queue_id = 'sitemap_' . md5($sitemap_url . time());
	1166	+ wp_schedule_single_event(time(), 'mxchat_process_sitemap_urls', array(
	1167	+ 'urls' => $urls,
	1168	+ 'sitemap_url' => $sitemap_url,
	1169	+ 'total_urls' => $total_urls,
	1170	+ 'batch_size' => absint(10),
	1171	+ 'batch_pause' => absint(5)
	1172	+ ));
1185	1173
1186		- // Add URLs to queue
1187		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'url', $urls, $bot_id);
	1174	+ $status_data = array(
	1175	+ 'total_urls' => $total_urls,
	1176	+ 'processed_urls' => 0,
	1177	+ 'status' => 'processing',
	1178	+ 'last_update' => time()
	1179	+ );
1188	1180
1189		- if ($queued_count === 0) {
1190		- throw new Exception(__('Failed to add URLs to processing queue', 'mxchat'));
1191		- }
	1181	+ set_transient(
	1182	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1183	+ array_map('sanitize_text_field', $status_data),
	1184	+ DAY_IN_SECONDS
	1185	+ );
1192	1186
1193		- // Store queue metadata
1194		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $sitemap_url);
1195		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'sitemap');
1196		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_urls);
1197		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
1198		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	1187	+ return __('scheduled', 'mxchat');
1199	1188
1200		- // Store queue ID in transient for status tracking
1201		- set_transient('mxchat_active_queue_sitemap', $queue_id, DAY_IN_SECONDS);
1202		- set_transient('mxchat_last_sitemap_url', $sitemap_url, DAY_IN_SECONDS);
1203		-
1204		- return 'queued';
1205		-
1206		- } catch (Exception $e) {
	1189	+ } catch (\Exception $e) {
1207	1190	$error_message = $e->getMessage();
1208	1191	//error_log(sprintf(esc_html__('Error preparing sitemap for processing: %s', 'mxchat'), esc_html($error_message)));
1209	1192
	1193	+ // Store the sitemap URL and error in transients so they can be displayed
	1194	+ set_transient(
	1195	+ 'mxchat_last_sitemap_url',
	1196	+ sanitize_text_field($sitemap_url),
	1197	+ DAY_IN_SECONDS
	1198	+ );
	1199	+
	1200	+ $status_data = array(
	1201	+ 'total_urls' => 0,
	1202	+ 'processed_urls' => 0,
	1203	+ 'status' => 'error',
	1204	+ 'error' => $error_message,
	1205	+ 'last_update' => time()
	1206	+ );
	1207	+
	1208	+ set_transient(
	1209	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1210	+ array_map('sanitize_text_field', $status_data),
	1211	+ DAY_IN_SECONDS
	1212	+ );
	1213	+
1210	1214	return $error_message;
1211	1215	}
1212		-
1213	1216	}
1214	1217
1215		-/**
1216		- * Remove shortcode tags but preserve the content inside them
1217		- * Example: [vc_column]Hello World[/vc_column] becomes "Hello World"
1218		- *
1219		- * @param string $content The content containing shortcodes
1220		- * @return string Content with shortcode tags removed but inner content preserved
1221		- */
1222		-private function strip_shortcode_tags_preserve_content($content) {
1223		- // Single-pass regex removes all shortcode brackets: [tag], [tag attr="val"], [/tag], [tag /]
1224		- // Content between tags is inherently preserved since only brackets are targeted
1225		- $result = preg_replace('/\[\/?\w[\w-]*[^\]]*\]/', '', $content);
1226		- return ($result !== null) ? $result : $content;
	1218	+public function mxchat_process_sitemap_urls_cron($urls, $sitemap_url, $total_urls, $batch_size, $batch_pause) {
	1219	+ // Validate inputs
	1220	+ $sitemap_url = esc_url_raw($sitemap_url);
	1221	+ $total_urls = absint($total_urls);
	1222	+ $batch_size = absint($batch_size);
	1223	+ $batch_pause = absint($batch_pause);
	1224	+
	1225	+ if (!is_array($urls) || empty($urls)) {
	1226	+ return;
	1227	+ }
	1228	+
	1229	+ try {
	1230	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	1231	+ $status = get_transient($status_key);
	1232	+
	1233	+ if (!$status || !is_array($status)) {
	1234	+ throw new Exception('Invalid status data retrieved from transient');
	1235	+ }
	1236	+
	1237	+ // Initialize failed_urls array if it doesn't exist
	1238	+ if (!isset($status['failed_urls_list']) || !is_array($status['failed_urls_list'])) {
	1239	+ $status['failed_urls_list'] = [];
	1240	+ }
	1241	+
	1242	+ $start_url = absint($status['processed_urls']);
	1243	+ $end_url = min($start_url + $batch_size, $total_urls);
	1244	+
	1245	+ // Track batch statistics
	1246	+ $batch_stats = [
	1247	+ 'processed' => 0,
	1248	+ 'failed' => 0,
	1249	+ 'last_error' => '',
	1250	+ 'embedding_errors' => 0
	1251	+ ];
	1252	+
	1253	+ // Check embedding configuration with first URL (only on first batch)
	1254	+ if ($start_url === 0) {
	1255	+ $test_url = esc_url_raw($urls[0]);
	1256	+ $test_response = wp_remote_get($test_url);
	1257	+
	1258	+ if (!is_wp_error($test_response) && wp_remote_retrieve_response_code($test_response) === 200) {
	1259	+ $test_html = wp_remote_retrieve_body($test_response);
	1260	+ $test_content = $this->mxchat_extract_main_content($test_html);
	1261	+ $test_sanitized = $this->mxchat_sanitize_content_for_api($test_content);
	1262	+
	1263	+ if (!empty($test_sanitized)) {
	1264	+ $test_embedding = $this->mxchat_generate_embedding($test_sanitized);
	1265	+
	1266	+ if (is_string($test_embedding)) {
	1267	+ throw new Exception('Embedding generation failed: ' . $test_embedding);
	1268	+ }
	1269	+
	1270	+ if (!is_array($test_embedding)) {
	1271	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($test_embedding));
	1272	+ }
	1273	+ }
	1274	+ }
	1275	+ }
	1276	+
	1277	+ for ($i = $start_url; $i < $end_url; $i++) {
	1278	+ $page_url = esc_url_raw($urls[$i]);
	1279	+ $max_retries = 3;
	1280	+ $retry_count = 0;
	1281	+ $url_processed = false;
	1282	+ $last_error = '';
	1283	+
	1284	+ while (!$url_processed && $retry_count < $max_retries) {
	1285	+ try {
	1286	+ // Attempt to fetch the URL
	1287	+ $page_response = wp_remote_get($page_url, array('timeout' => 30));
	1288	+
	1289	+ if (is_wp_error($page_response)) {
	1290	+ throw new Exception('HTTP request failed: ' . $page_response->get_error_message());
	1291	+ }
	1292	+
	1293	+ $response_code = wp_remote_retrieve_response_code($page_response);
	1294	+ if ($response_code !== 200) {
	1295	+ throw new Exception('HTTP Status: ' . $response_code);
	1296	+ }
	1297	+
	1298	+ $page_html = wp_remote_retrieve_body($page_response);
	1299	+
	1300	+ if (empty($page_html)) {
	1301	+ throw new Exception('Empty response body');
	1302	+ }
	1303	+
	1304	+ $page_content = $this->mxchat_extract_main_content($page_html);
	1305	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	1306	+
	1307	+ if (empty($sanitized_content)) {
	1308	+ throw new Exception('No valid content found after processing');
	1309	+ }
	1310	+
	1311	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	1312	+
	1313	+ if (is_string($embedding_vector)) {
	1314	+ throw new Exception('Embedding generation failed: ' . $embedding_vector);
	1315	+ }
	1316	+
	1317	+ if (!is_array($embedding_vector)) {
	1318	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($embedding_vector));
	1319	+ }
	1320	+
	1321	+ // Submit to database
	1322	+ $options = get_option('mxchat_options');
	1323	+ $submission_result = MxChat_Utils::submit_content_to_db($sanitized_content, $page_url, $options['api_key']);
	1324	+
	1325	+ if (is_wp_error($submission_result)) {
	1326	+ throw new Exception('Database submission failed: ' . $submission_result->get_error_message());
	1327	+ }
	1328	+
	1329	+ // Success!
	1330	+ $url_processed = true;
	1331	+ $batch_stats['processed']++;
	1332	+
	1333	+ } catch (Exception $e) {
	1334	+ $retry_count++;
	1335	+ $last_error = $e->getMessage();
	1336	+
	1337	+ //error_log("URL {$page_url} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	1338	+
	1339	+ // Track embedding errors specifically
	1340	+ if (strpos($last_error, 'Embedding') !== false) {
	1341	+ $batch_stats['embedding_errors']++;
	1342	+ }
	1343	+
	1344	+ if ($retry_count < $max_retries) {
	1345	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	1346	+ sleep(pow(2, $retry_count - 1));
	1347	+ }
	1348	+ }
	1349	+ }
	1350	+
	1351	+ // If URL still not processed after all retries, mark as failed
	1352	+ if (!$url_processed) {
	1353	+ $batch_stats['failed']++;
	1354	+ $batch_stats['last_error'] = $last_error;
	1355	+
	1356	+ // Add to failed URLs list
	1357	+ $status['failed_urls_list'][] = [
	1358	+ 'url' => $page_url,
	1359	+ 'error' => $last_error,
	1360	+ 'time' => time(),
	1361	+ 'retries' => $max_retries
	1362	+ ];
	1363	+
	1364	+ // Limit the number of failed URLs we store to prevent transient size issues
	1365	+ if (count($status['failed_urls_list']) > 100) {
	1366	+ $status['failed_urls_list'] = array_slice($status['failed_urls_list'], -100);
	1367	+ }
	1368	+ }
	1369	+
	1370	+ // Update progress
	1371	+ $status['processed_urls'] = absint($i + 1);
	1372	+ $status['last_update'] = time();
	1373	+ $status['failed_urls'] = absint($status['failed_urls'] ?? 0) + ($url_processed ? 0 : 1);
	1374	+ $status['last_error'] = $batch_stats['last_error'];
	1375	+
	1376	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1377	+
	1378	+ // If we have too many consecutive embedding errors, stop processing
	1379	+ if ($batch_stats['embedding_errors'] >= 10) {
	1380	+ throw new Exception('Too many consecutive embedding failures detected. Please check your API configuration.');
	1381	+ }
	1382	+ }
	1383	+
	1384	+ // If all URLs in this batch failed, stop processing
	1385	+ if ($batch_stats['processed'] === 0 && $batch_stats['failed'] > 0) {
	1386	+ $status['status'] = 'error';
	1387	+ $status['error'] = sprintf(
	1388	+ 'Processing stopped: %d consecutive failures in batch. Last error: %s',
	1389	+ $batch_stats['failed'],
	1390	+ $batch_stats['last_error']
	1391	+ );
	1392	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1393	+ return;
	1394	+ }
	1395	+
	1396	+ // Update final progress
	1397	+ $status['processed_urls'] = min($end_url, $total_urls);
	1398	+ $status['last_update'] = time();
	1399	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1400	+
	1401	+ // Check if we've processed all URLs
	1402	+ if ($end_url >= $total_urls) {
	1403	+ // All URLs have been processed - mark as complete
	1404	+ $status['status'] = 'complete';
	1405	+ $status['processed_urls'] = $total_urls;
	1406	+
	1407	+ // Add completion summary
	1408	+ $status['completion_summary'] = [
	1409	+ 'total_urls' => $total_urls,
	1410	+ 'successful_urls' => $total_urls - absint($status['failed_urls'] ?? 0),
	1411	+ 'failed_urls' => absint($status['failed_urls'] ?? 0),
	1412	+ 'completion_time' => current_time('mysql')
	1413	+ ];
	1414	+
	1415	+ // Save the completed status (don't delete it - let user dismiss manually)
	1416	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1417	+
	1418	+ // DON'T delete the status transients here - let user dismiss manually
	1419	+ } else {
	1420	+ // Schedule next batch
	1421	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_sitemap_urls', array(
	1422	+ 'urls' => $urls,
	1423	+ 'sitemap_url' => $sitemap_url,
	1424	+ 'total_urls' => $total_urls,
	1425	+ 'batch_size' => $batch_size,
	1426	+ 'batch_pause' => $batch_pause,
	1427	+ ));
	1428	+ }
	1429	+ } catch (\Exception $e) {
	1430	+ $status['status'] = 'error';
	1431	+ $status['error'] = $e->getMessage();
	1432	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1433	+ }
1227	1434	}
1228		-
1229	1435	public function mxchat_sanitize_content_for_api($content) {
1230	1436	//error_log('[MXCHAT-SANITIZE] Original content preview: ' . substr($content, 0, 500) . '...');
1231		-
1232		- // Remove shortcode tags but PRESERVE content inside them
1233		- $content = $this->strip_shortcode_tags_preserve_content($content);
1234		-
	1437	+
1235	1438	// Remove script, style tags, and HTML comments
1236	1439	$content = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $content);
1237	1440	$content = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $content);
1238	1441	$content = preg_replace('/<!--(.|\s)*?-->/', '', $content);
1239		-
	1442	+
1240	1443	// Remove all HTML tags and decode HTML entities
1241	1444	$content = wp_strip_all_tags($content);
1242	1445	$content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5);
1243	1446
		@@ -1295,12 +1498,12 @@
1295	1498	@$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'), LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
1296	1499	$xpath = new DOMXPath($dom);
1297	1500
1298	1501	// For debugging purposes
1299		- $debugEnabled = true; // Set to true to enable debugging output
	1502	+ $debugEnabled = false; // Set to true to enable debugging output
1300	1503	$debug = function($message) use ($debugEnabled) {
1301	1504	if ($debugEnabled) {
1302		- //error_log('[MXCHAT-EXTRACT-DEBUG] ' . $message);
	1505	+ //error_log('[MXCHAT-DEBUG] ' . $message);
1303	1506	}
1304	1507	};
1305	1508
1306	1509	// Direct targeting for Gerow theme posts
		@@ -1399,25 +1602,18 @@
1399	1602	'//main',
1400	1603	'//div[contains(@class, "content")]'
1401	1604	];
1402	1605
1403		- // First handle Elementor content - get only leaf widget containers to avoid duplicates
	1606	+ // First handle Elementor content
1404	1607	$debug("Checking for Elementor content");
1405		- // Get widget containers that are direct children of widgets (not nested inside other widget containers)
1406		- $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-widget")]//div[contains(@class, "elementor-widget-container")]');
	1608	+ $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-element")]//div[contains(@class, "elementor-widget-container")]');
1407	1609	if ($elementor_widgets && $elementor_widgets->length > 0) {
1408	1610	$debug("Found Elementor widgets");
1409		- $seen_content = array(); // Track seen content to avoid duplicates
1410	1611	$combined_content = '';
1411	1612	foreach ($elementor_widgets as $widget) {
1412	1613	$widget_content = $dom->saveHTML($widget);
1413	1614	if (!empty($widget_content)) {
1414		- // Create a hash of the content to detect duplicates
1415		- $content_hash = md5($widget_content);
1416		- if (!isset($seen_content[$content_hash])) {
1417		- $seen_content[$content_hash] = true;
1418		- $combined_content .= $widget_content;
1419		- }
	1615	+ $combined_content .= $widget_content;
1420	1616	}
1421	1617	}
1422	1618	if (!empty($combined_content)) {
1423	1619	$debug("Returning Elementor content");
		@@ -1429,14 +1625,15 @@
1429	1625	foreach ($selectors as $selector) {
1430	1626	$debug("Trying selector: " . $selector);
1431	1627	$nodes = $xpath->query($selector);
1432	1628	if ($nodes && $nodes->length > 0) {
1433		- $debug("Found " . $nodes->length . " matches for selector: " . $selector);
1434		- // Only take the FIRST matching node to avoid duplicate content
1435		- // (pages often have nested or multiple containers with same class)
1436		- $content = $dom->saveHTML($nodes->item(0));
	1629	+ $debug("Found matches for selector: " . $selector);
	1630	+ $content = '';
	1631	+ foreach ($nodes as $node) {
	1632	+ $content .= $dom->saveHTML($node);
	1633	+ }
1437	1634	if (!empty($content)) {
1438		- $debug("Returning content from selector: " . $selector . " (first match only)");
	1635	+ $debug("Returning content from selector: " . $selector);
1439	1636	return $content;
1440	1637	}
1441	1638	}
1442	1639	}
		@@ -1460,106 +1657,16 @@
1460	1657	$debug("Returning blog-area section content");
1461	1658	return $content;
1462	1659	}
1463	1660	}
1464		-
1465		- // Generic container selectors for non-CMS sites (like .asp pages)
1466		- $debug("Trying generic container selectors");
1467		- $generic_selectors = [
1468		- '//div[@id="main"]',
1469		- '//div[@id="wrapper"]',
1470		- '//div[@id="page"]',
1471		- '//div[@id="site-content"]',
1472		- '//div[contains(@class, "main-content")]',
1473		- '//div[contains(@class, "page-content")]',
1474		- '//div[contains(@class, "site-content")]',
1475		- ];
1476		-
1477		- foreach ($generic_selectors as $selector) {
1478		- $debug("Trying generic selector: " . $selector);
1479		- $nodes = $xpath->query($selector);
1480		- if ($nodes && $nodes->length > 0) {
1481		- $content = $dom->saveHTML($nodes->item(0));
1482		- if (!empty($content)) {
1483		- $debug("Returning content from generic selector: " . $selector);
1484		- return $content;
1485		- }
1486		- }
1487		- }
1488		-
1489		- // Paragraph-based content detection - find regions with substantial text
1490		- $debug("Trying paragraph-based content detection");
1491		- $paragraphs = $xpath->query('//p[string-length(normalize-space()) > 50]');
1492		- if ($paragraphs && $paragraphs->length >= 3) {
1493		- $debug("Found " . $paragraphs->length . " substantial paragraphs");
1494		- // Collect all substantial paragraphs and their content
1495		- $paragraph_content = '';
1496		- foreach ($paragraphs as $p) {
1497		- $paragraph_content .= $dom->saveHTML($p) . "\n";
1498		- }
1499		- if (!empty($paragraph_content)) {
1500		- $debug("Returning paragraph-based content");
1501		- return $paragraph_content;
1502		- }
1503		- }
1504		-
1505		- // Improved body fallback - strip nav/header/footer elements first
1506		- $debug("Using improved body fallback");
	1661	+
	1662	+ // Fallback: Return the body content if no specific selector matches
	1663	+ $debug("Using body fallback");
1507	1664	$body = $dom->getElementsByTagName('body');
1508	1665	if ($body->length > 0) {
1509		- // Clone the body to avoid modifying the original DOM
1510		- $body_clone = $body->item(0)->cloneNode(true);
1511		-
1512		- // Remove common non-content elements by tag name
1513		- $remove_tags = ['nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript'];
1514		- foreach ($remove_tags as $tag) {
1515		- $elements = $body_clone->getElementsByTagName($tag);
1516		- // Iterate backwards to safely remove elements
1517		- for ($i = $elements->length - 1; $i >= 0; $i--) {
1518		- $el = $elements->item($i);
1519		- if ($el && $el->parentNode) {
1520		- $el->parentNode->removeChild($el);
1521		- }
1522		- }
1523		- }
1524		-
1525		- // Remove elements with common non-content class names using XPath on the cloned body
1526		- $temp_dom = new DOMDocument();
1527		- @$temp_dom->appendChild($temp_dom->importNode($body_clone, true));
1528		- $temp_xpath = new DOMXPath($temp_dom);
1529		-
1530		- $remove_class_patterns = [
1531		- '//*[contains(@class, "nav")]',
1532		- '//*[contains(@class, "menu")]',
1533		- '//*[contains(@class, "sidebar")]',
1534		- '//*[contains(@class, "footer")]',
1535		- '//*[contains(@class, "header")]',
1536		- '//*[contains(@id, "nav")]',
1537		- '//*[contains(@id, "menu")]',
1538		- '//*[contains(@id, "sidebar")]',
1539		- '//*[contains(@id, "footer")]',
1540		- '//*[contains(@id, "header")]',
1541		- ];
1542		-
1543		- foreach ($remove_class_patterns as $pattern) {
1544		- $elements = $temp_xpath->query($pattern);
1545		- if ($elements) {
1546		- for ($i = $elements->length - 1; $i >= 0; $i--) {
1547		- $el = $elements->item($i);
1548		- if ($el && $el->parentNode) {
1549		- $el->parentNode->removeChild($el);
1550		- }
1551		- }
1552		- }
1553		- }
1554		-
1555		- $cleaned_content = $temp_dom->saveHTML();
1556		- if (!empty($cleaned_content)) {
1557		- $debug("Returning cleaned body content");
1558		- return $cleaned_content;
1559		- }
	1666	+ return $dom->saveHTML($body->item(0));
1560	1667	}
1561		-
	1668	+
1562	1669	// Last resort: return the original HTML
1563	1670	$debug("Returning original HTML");
1564	1671	return $html;
1565	1672	} catch (Exception $e) {
		@@ -1611,41 +1718,47 @@
1611	1718	try {
1612	1719	// Verify the request
1613	1720	check_ajax_referer('mxchat_status_nonce', 'nonce');
1614	1721
1615		- // Get active queue IDs
1616		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
1617		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
	1722	+ // Get the status just like in your admin page
	1723	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1724	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
1618	1725
1619		- $sitemap_status = false;
1620		- $pdf_status = false;
	1726	+ $pdf_status = $pdf_url ? $this->mxchat_get_pdf_processing_status($pdf_url) : false;
	1727	+ $sitemap_status = $sitemap_url ? $this->mxchat_get_sitemap_processing_status($sitemap_url) : false;
1621	1728
1622		- // Get sitemap queue status
1623		- if ($sitemap_queue_id) {
1624		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
	1729	+ // Add the PDF URL to the status object
	1730	+ if ($pdf_status && $pdf_url) {
	1731	+ $pdf_status['pdf_url'] = $pdf_url;
1625	1732	}
1626	1733
1627		- // Get PDF queue status
1628		- if ($pdf_queue_id) {
1629		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
1630		- }
	1734	+ // Set the current PDF URL for the manual batch processing button
	1735	+ $current_pdf_url = $pdf_url;
1631	1736
	1737	+ // Check for true processing status, not just presence of status
1632	1738	$is_active_processing =
1633		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
1634		- ($pdf_status && $pdf_status['status'] === 'processing');
	1739	+ ($sitemap_status && isset($sitemap_status['status']) && $sitemap_status['status'] === 'processing') ||
	1740	+ ($pdf_status && isset($pdf_status['status']) && $pdf_status['status'] === 'processing');
1635	1741
	1742	+ // Get single URL status, but only if no processing is active
	1743	+ $single_url_status = !$is_active_processing ? $this->mxchat_get_single_url_status() : false;
	1744	+
	1745	+ // REMOVED: Auto-clearing of completed status - now only done via dismiss button
	1746	+
1636	1747	// Return JSON response with the status data
1637	1748	wp_send_json(array(
1638	1749	'pdf_status' => $pdf_status,
1639	1750	'sitemap_status' => $sitemap_status,
	1751	+ 'single_url_status' => $single_url_status,
1640	1752	'is_processing' => $is_active_processing,
1641		- 'sitemap_queue_id' => $sitemap_queue_id,
1642		- 'pdf_queue_id' => $pdf_queue_id
	1753	+ 'current_pdf_url' => $current_pdf_url
1643	1754	));
1644	1755
1645	1756	} catch (Exception $e) {
	1757	+ // Log the error
1646	1758	//error_log('MxChat Status Update Error: ' . $e->getMessage());
1647	1759
	1760	+ // Return a friendly error response
1648	1761	wp_send_json_error(array(
1649	1762	'message' => 'Error getting status updates: ' . $e->getMessage(),
1650	1763	'status' => 'error'
1651	1764	));
		@@ -1650,1373 +1763,8 @@
1650	1763	'status' => 'error'
1651	1764	));
1652	1765	}
1653	1766	}
1654		-
1655		-/**
1656		- * Helper function to get queue status data
1657		- */
1658		-private function mxchat_get_queue_status_data($queue_id, $type = 'sitemap') {
1659		- global $wpdb;
1660		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
1661		-
1662		- // Get counts by status
1663		- $counts = $wpdb->get_results($wpdb->prepare(
1664		- "SELECT status, COUNT(*) as count
1665		- FROM $table_name
1666		- WHERE queue_id = %s
1667		- GROUP BY status",
1668		- $queue_id
1669		- ), OBJECT_K);
1670		-
1671		- $total = 0;
1672		- $completed = 0;
1673		- $failed = 0;
1674		- $processing = 0;
1675		- $pending = 0;
1676		-
1677		- foreach ($counts as $status => $data) {
1678		- $count = absint($data->count);
1679		- $total += $count;
1680		-
1681		- switch ($status) {
1682		- case 'completed':
1683		- $completed = $count;
1684		- break;
1685		- case 'failed':
1686		- $failed = $count;
1687		- break;
1688		- case 'processing':
1689		- $processing = $count;
1690		- break;
1691		- case 'pending':
1692		- $pending = $count;
1693		- break;
1694		- }
1695		- }
1696		-
1697		- if ($total === 0) {
1698		- return false;
1699		- }
1700		-
1701		- // Calculate percentage
1702		- $percentage = round((($completed + $failed) / $total) * 100);
1703		-
1704		- // Get failed items details (limit to 50)
1705		- $failed_items = array();
1706		- if ($failed > 0) {
1707		- $failed_results = $wpdb->get_results($wpdb->prepare(
1708		- "SELECT item_type, item_data, error_message, attempts, completed_at
1709		- FROM $table_name
1710		- WHERE queue_id = %s
1711		- AND status = 'failed'
1712		- AND attempts >= max_attempts
1713		- ORDER BY id DESC
1714		- LIMIT 50",
1715		- $queue_id
1716		- ));
1717		-
1718		- foreach ($failed_results as $item) {
1719		- $data = json_decode($item->item_data, true);
1720		- $url = $type === 'pdf' ? ($data['pdf_url'] ?? '') : ($data['url'] ?? '');
1721		-
1722		- $failed_items[] = array(
1723		- 'url' => $url,
1724		- 'page' => $type === 'pdf' ? ($data['page_number'] ?? 0) : 0,
1725		- 'error' => $item->error_message,
1726		- 'retries' => $item->attempts,
1727		- 'time' => strtotime($item->completed_at)
1728		- );
1729		- }
1730		- }
1731		-
1732		- // Get queue metadata
1733		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
1734		-
1735		- // Determine if queue is complete
1736		- $is_complete = ($pending === 0 && $processing === 0);
1737		-
1738		- // Get last update time
1739		- $last_update = $wpdb->get_var($wpdb->prepare(
1740		- "SELECT MAX(UNIX_TIMESTAMP(COALESCE(completed_at, started_at, created_at)))
1741		- FROM $table_name
1742		- WHERE queue_id = %s",
1743		- $queue_id
1744		- ));
1745		-
1746		- $last_update_text = $last_update ? human_time_diff($last_update, time()) . ' ' . __('ago', 'mxchat') : __('Just now', 'mxchat');
1747		-
1748		- // Format based on type
1749		- if ($type === 'pdf') {
1750		- return array(
1751		- 'total_pages' => $total,
1752		- 'processed_pages' => $completed + $failed,
1753		- 'failed_pages' => $failed,
1754		- 'percentage' => $percentage,
1755		- 'status' => $is_complete ? 'complete' : 'processing',
1756		- 'last_update' => $last_update_text,
1757		- 'failed_pages_list' => $failed_items,
1758		- 'pdf_url' => $source_url,
1759		- 'queue_id' => $queue_id
1760		- );
1761		- } else {
1762		- return array(
1763		- 'total_urls' => $total,
1764		- 'processed_urls' => $completed + $failed,
1765		- 'failed_urls' => $failed,
1766		- 'percentage' => $percentage,
1767		- 'status' => $is_complete ? 'complete' : 'processing',
1768		- 'last_update' => $last_update_text,
1769		- 'failed_urls_list' => $failed_items,
1770		- 'sitemap_url' => $source_url,
1771		- 'queue_id' => $queue_id
1772		- );
1773		- }
1774		-}
1775		-
1776		-/**
1777		- * Public method to get processing status for both sitemap and PDF queues
1778		- * Used by admin pages to display processing status
1779		- *
1780		- * @return array Array with 'sitemap_status', 'pdf_status', and 'is_processing' keys
1781		- */
1782		-public function mxchat_get_processing_statuses() {
1783		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
1784		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
1785		-
1786		- $sitemap_status = false;
1787		- $pdf_status = false;
1788		-
1789		- if ($sitemap_queue_id) {
1790		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
1791		- }
1792		-
1793		- if ($pdf_queue_id) {
1794		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
1795		- }
1796		-
1797		- $is_processing =
1798		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
1799		- ($pdf_status && $pdf_status['status'] === 'processing');
1800		-
1801		- return array(
1802		- 'sitemap_status' => $sitemap_status,
1803		- 'pdf_status' => $pdf_status,
1804		- 'is_processing' => $is_processing
1805		- );
1806		-}
1807		-
1808		-/**
1809		- * AJAX handler to get recent knowledge entries for real-time table updates
1810		- * UPDATED: Now supports both WordPress DB and Pinecone data sources
1811		- */
1812		-public function ajax_mxchat_get_recent_entries() {
1813		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
1814		-
1815		- if (!current_user_can('manage_options')) {
1816		- wp_send_json_error(array('message' => 'Unauthorized'));
1817		- return;
1818		- }
1819		-
1820		- global $wpdb;
1821		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1822		-
1823		- // Get parameters
1824		- $last_id = isset($_POST['last_id']) ? absint($_POST['last_id']) : 0;
1825		- $limit = isset($_POST['limit']) ? min(absint($_POST['limit']), 50) : 10;
1826		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
1827		-
1828		- // Check if Pinecone is enabled for this bot
1829		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
1830		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
1831		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
1832		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
1833		-
1834		- if ($use_pinecone && $has_pinecone_api) {
1835		- // PINECONE DATA SOURCE - Get grouped entry count (not raw vector count)
1836		- // Use mxchat_fetch_pinecone_records which returns total_unique_entries
1837		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, '', 1, 10, $bot_id, '');
1838		- $total_count = $records['total'] ?? 0;
1839		-
1840		- // For Pinecone, we don't return individual entries during polling
1841		- // (entries are already displayed on page load via mxchat_fetch_pinecone_records)
1842		- // We just return the updated count
1843		- wp_send_json_success(array(
1844		- 'entries' => array(),
1845		- 'total_count' => absint($total_count),
1846		- 'max_id' => $last_id,
1847		- 'data_source' => 'pinecone'
1848		- ));
1849		- return;
1850		- }
1851		-
1852		- // WORDPRESS DB DATA SOURCE
1853		- // Build query to get entries newer than last_id
1854		- $where_clauses = array('1=1');
1855		- $where_values = array();
1856		-
1857		- if ($last_id > 0) {
1858		- $where_clauses[] = 'id > %d';
1859		- $where_values[] = $last_id;
1860		- }
1861		-
1862		- // Note: WordPress DB table doesn't have bot_id column
1863		- // Multi-bot filtering is handled via Pinecone namespaces
1864		-
1865		- $where_sql = implode(' AND ', $where_clauses);
1866		-
1867		- // Get recent entries
1868		- $query = "SELECT id, article_content, source_url, timestamp
1869		- FROM $table_name
1870		- WHERE $where_sql
1871		- ORDER BY id DESC
1872		- LIMIT %d";
1873		-
1874		- $where_values[] = $limit;
1875		-
1876		- $entries = $wpdb->get_results($wpdb->prepare($query, $where_values));
1877		-
1878		- // Get total count of GROUPED entries (by source_url) - matches pagination display
1879		- // Count unique source_urls (excluding mxchat:// internal refs) + count of ungrouped rows
1880		- $total_count = $wpdb->get_var(
1881		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
1882		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
1883		- );
1884		-
1885		- // Format entries for response
1886		- $formatted_entries = array();
1887		- $preview_length = 150;
1888		- foreach ($entries as $entry) {
1889		- // Parse chunk metadata using the proper chunker method (same as initial page load)
1890		- if (class_exists('MxChat_Chunker')) {
1891		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($entry->article_content);
1892		- $display_content = $chunk_meta['text'];
1893		- $chunk_metadata = $chunk_meta['metadata'];
1894		- } else {
1895		- $display_content = $entry->article_content;
1896		- $chunk_metadata = array();
1897		- }
1898		-
1899		- $content_preview = mb_strlen($display_content) > $preview_length
1900		- ? mb_substr($display_content, 0, $preview_length) . '...'
1901		- : $display_content;
1902		-
1903		- $formatted_entries[] = array(
1904		- 'id' => $entry->id,
1905		- 'preview' => esc_html($content_preview),
1906		- 'full_content' => wp_kses_post(wpautop($display_content)),
1907		- 'content_length' => mb_strlen($display_content),
1908		- 'preview_length' => $preview_length,
1909		- 'source_url' => $entry->source_url,
1910		- 'has_link' => !empty($entry->source_url) && strpos($entry->source_url, 'mxchat://') !== 0,
1911		- 'chunk_metadata' => $chunk_metadata,
1912		- 'bot_id' => $entry->bot_id ?? 'default',
1913		- 'edit_nonce' => wp_create_nonce('mxchat_edit_entry_nonce'),
1914		- 'delete_nonce' => wp_create_nonce('mxchat_delete_prompt_nonce')
1915		- );
1916		- }
1917		-
1918		- wp_send_json_success(array(
1919		- 'entries' => $formatted_entries,
1920		- 'total_count' => absint($total_count),
1921		- 'max_id' => !empty($entries) ? $entries[0]->id : $last_id,
1922		- 'data_source' => 'wordpress'
1923		- ));
1924		-}
1925		-
1926		-/**
1927		- * Get Pinecone total count from stats API
1928		- * Helper function for ajax_mxchat_get_recent_entries
1929		- */
1930		-private function mxchat_get_pinecone_count_from_stats($pinecone_options) {
1931		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1932		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1933		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1934		-
1935		- if (empty($api_key) || empty($host)) {
1936		- return 0;
1937		- }
1938		-
1939		- try {
1940		- $stats_url = "https://{$host}/describe_index_stats";
1941		-
1942		- $response = wp_remote_post($stats_url, array(
1943		- 'headers' => array(
1944		- 'Api-Key' => $api_key,
1945		- 'Content-Type' => 'application/json'
1946		- ),
1947		- 'body' => '{}',
1948		- 'timeout' => 10
1949		- ));
1950		-
1951		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
1952		- $body = wp_remote_retrieve_body($response);
1953		- $stats_data = json_decode($body, true);
1954		-
1955		- // If namespace is specified, get count from that specific namespace
1956		- if (!empty($namespace) && isset($stats_data['namespaces'][$namespace]['vectorCount'])) {
1957		- return intval($stats_data['namespaces'][$namespace]['vectorCount']);
1958		- }
1959		-
1960		- // If no namespace specified or namespace not found in response, use total
1961		- return intval($stats_data['totalVectorCount'] ?? 0);
1962		- }
1963		-
1964		- return 0;
1965		-
1966		- } catch (Exception $e) {
1967		- return 0;
1968		- }
1969		-}
1970		-
1971		-/**
1972		- * AJAX handler to refresh Pinecone entries table via AJAX
1973		- * Returns the table HTML for updating the UI without a full page reload
1974		- */
1975		-public function ajax_mxchat_refresh_pinecone_entries() {
1976		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
1977		-
1978		- if (!current_user_can('manage_options')) {
1979		- wp_send_json_error(array('message' => 'Unauthorized'));
1980		- return;
1981		- }
1982		-
1983		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
1984		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
1985		- $per_page = 25;
1986		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
1987		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
1988		-
1989		- // Get Pinecone manager and options
1990		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
1991		- if (!$pinecone_manager) {
1992		- wp_send_json_error(array('message' => 'Pinecone manager not available'));
1993		- return;
1994		- }
1995		-
1996		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
1997		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
1998		- $pinecone_api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1999		-
2000		- if (!$use_pinecone || empty($pinecone_api_key)) {
2001		- wp_send_json_error(array('message' => 'Pinecone not configured'));
2002		- return;
2003		- }
2004		-
2005		- // Fetch records from Pinecone
2006		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, $search_query, $page, $per_page, $bot_id, $content_type_filter);
2007		- $prompts = $records['data'] ?? array();
2008		- $total_records = $records['total'] ?? 0;
2009		-
2010		- // Preprocess Pinecone records — set chunk_metadata and display_content
2011		- // (matches admin-knowledge-page.php preprocessing)
2012		- foreach ($prompts as $prompt) {
2013		- if (isset($prompt->chunk_index) && $prompt->chunk_index !== null) {
2014		- $prompt->chunk_metadata = array(
2015		- 'chunk_index' => intval($prompt->chunk_index),
2016		- 'total_chunks' => isset($prompt->total_chunks) ? intval($prompt->total_chunks) : null,
2017		- 'is_chunked' => isset($prompt->is_chunked) ? (bool) $prompt->is_chunked : true,
2018		- 'source_url' => $prompt->source_url ?? ''
2019		- );
2020		- $prompt->display_content = $prompt->article_content;
2021		- } else {
2022		- $prompt->chunk_metadata = array();
2023		- $prompt->display_content = $prompt->article_content ?? '';
2024		- }
2025		- }
2026		-
2027		- // Group prompts by source_url
2028		- $grouped_prompts = array();
2029		- foreach ($prompts as $prompt) {
2030		- $source_url = '';
2031		- if (!empty($prompt->chunk_metadata['source_url'])) {
2032		- $source_url = $prompt->chunk_metadata['source_url'];
2033		- } elseif (!empty($prompt->source_url)) {
2034		- $source_url = $prompt->source_url;
2035		- }
2036		-
2037		- if (!empty($source_url)) {
2038		- if (!isset($grouped_prompts[$source_url])) {
2039		- $grouped_prompts[$source_url] = array();
2040		- }
2041		- $grouped_prompts[$source_url][] = $prompt;
2042		- } else {
2043		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2044		- }
2045		- }
2046		-
2047		- // Sort each group by chunk_index
2048		- foreach ($grouped_prompts as $source_url => &$group) {
2049		- usort($group, function($a, $b) {
2050		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2051		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2052		- return $index_a - $index_b;
2053		- });
2054		- }
2055		- unset($group);
2056		-
2057		- // Build HTML for the table rows - must match admin-knowledge-page.php structure exactly
2058		- ob_start();
2059		- $display_index = 0;
2060		- $current_page = $page;
2061		- $data_source = 'pinecone';
2062		- $current_bot_id = $bot_id;
2063		- $preview_length = 150;
2064		-
2065		- if (empty($grouped_prompts)) {
2066		- echo '<tr><td colspan="4" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2067		- esc_html_e('No knowledge entries found in Pinecone.', 'mxchat');
2068		- echo '</td></tr>';
2069		- } else {
2070		- foreach ($grouped_prompts as $source_url => $group) {
2071		- $chunk_count = count($group);
2072		- $first_prompt = $group[0];
2073		- $display_index++;
2074		-
2075		- if ($chunk_count > 1) {
2076		- // Multiple chunks - show grouped row with expand button
2077		- $group_id = 'group-' . md5($source_url);
2078		- ?>
2079		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2080		- class="mxchat-chunk-group-header"
2081		- data-source="<?php echo esc_attr($data_source); ?>"
2082		- data-group-id="<?php echo esc_attr($group_id); ?>"
2083		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2084		- <td style="padding: 12px 16px; text-align: center;">
2085		- <input type="checkbox"
2086		- class="mxchat-entry-checkbox"
2087		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2088		- data-source="<?php echo esc_attr($data_source); ?>"
2089		- data-source-url="<?php echo esc_attr($source_url); ?>"
2090		- data-is-group="true"
2091		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2092		- </td>
2093		- <td style="padding: 12px 16px; font-size: 13px;">
2094		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2095		- </td>
2096		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2097		- <div class="mxchat-chunk-group-info">
2098		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2099		- <span class="dashicons dashicons-arrow-right-alt2"></span>
2100		- </button>
2101		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2102		- <span class="mxchat-chunk-preview">
2103		- <?php
2104		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : (isset($first_prompt->article_content) ? $first_prompt->article_content : '');
2105		- $content_preview = mb_substr($parent_content, 0, 100);
2106		- echo esc_html($content_preview . '...');
2107		- ?>
2108		- </span>
2109		- </div>
2110		- </td>
2111		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2112		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2113		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2114		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2115		- <?php esc_html_e('View Source', 'mxchat'); ?>
2116		- </a>
2117		- <?php else : ?>
2118		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2119		- <?php endif; ?>
2120		- </td>
2121		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2122		- <?php if ($data_source !== 'pinecone') : ?>
2123		- <button type="button"
2124		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2125		- data-source-url="<?php echo esc_attr($source_url); ?>"
2126		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2127		- data-data-source="<?php echo esc_attr($data_source); ?>"
2128		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2129		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2130		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2131		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2132		- </button>
2133		- <?php endif; ?>
2134		- <button type="button"
2135		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2136		- data-source-url="<?php echo esc_attr($source_url); ?>"
2137		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2138		- data-data-source="<?php echo esc_attr($data_source); ?>"
2139		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2140		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2141		- style="color: var(--mxch-error);"
2142		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2143		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2144		- </button>
2145		- </td>
2146		- </tr>
2147		- <?php
2148		- // Render hidden chunk rows
2149		- foreach ($group as $chunk_index => $chunk) {
2150		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
2151		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
2152		- $content = isset($chunk->display_content) ? $chunk->display_content : (isset($chunk->article_content) ? $chunk->article_content : '');
2153		- $content_preview = mb_strlen($content) > $preview_length
2154		- ? mb_substr($content, 0, $preview_length) . '...'
2155		- : $content;
2156		- ?>
2157		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
2158		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
2159		- data-source="<?php echo esc_attr($data_source); ?>"
2160		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
2161		- <td style="padding: 12px 16px; text-align: center;">
2162		- <!-- Checkbox column placeholder for chunks (managed by group) -->
2163		- </td>
2164		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
2165		- <!-- Hidden ID column for chunks -->
2166		- </td>
2167		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2168		- <div class="mxchat-accordion-wrapper">
2169		- <div class="mxchat-content-preview">
2170		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
2171		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
2172		- </span>
2173		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2174		- <?php if (mb_strlen($content) > $preview_length) : ?>
2175		- <button class="mxchat-expand-toggle" type="button">
2176		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2177		- </button>
2178		- <?php endif; ?>
2179		- </div>
2180		- <div class="mxchat-content-full" style="display: none;">
2181		- <div class="content-view">
2182		- <?php
2183		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2184		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2185		- echo wp_kses_post(wpautop($content));
2186		- echo '</div>';
2187		- } else {
2188		- echo wp_kses_post(wpautop($content));
2189		- }
2190		- ?>
2191		- </div>
2192		- </div>
2193		- </div>
2194		- </td>
2195		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2196		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
2197		- </td>
2198		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
2199		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
2200		- </td>
2201		- </tr>
2202		- <?php
2203		- }
2204		- } else {
2205		- // Single entry - display normally with accordion
2206		- $prompt = $first_prompt;
2207		- $content = isset($prompt->display_content) ? $prompt->display_content : (isset($prompt->article_content) ? $prompt->article_content : '');
2208		- $content_preview = mb_strlen($content) > $preview_length
2209		- ? mb_substr($content, 0, $preview_length) . '...'
2210		- : $content;
2211		- ?>
2212		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
2213		- data-source="<?php echo esc_attr($data_source); ?>"
2214		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2215		- <td style="padding: 12px 16px; text-align: center;">
2216		- <input type="checkbox"
2217		- class="mxchat-entry-checkbox"
2218		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2219		- data-source="<?php echo esc_attr($data_source); ?>"
2220		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
2221		- data-is-group="false"
2222		- data-chunk-count="1">
2223		- </td>
2224		- <td style="padding: 12px 16px; font-size: 13px;">
2225		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2226		- </td>
2227		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2228		- <div class="mxchat-accordion-wrapper">
2229		- <div class="mxchat-content-preview">
2230		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2231		- <?php if (mb_strlen($content) > $preview_length) : ?>
2232		- <button class="mxchat-expand-toggle" type="button">
2233		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2234		- </button>
2235		- <?php endif; ?>
2236		- </div>
2237		- <div class="mxchat-content-full" style="display: none;">
2238		- <div class="content-view">
2239		- <?php
2240		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2241		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2242		- echo wp_kses_post(wpautop($content));
2243		- echo '</div>';
2244		- } else {
2245		- echo wp_kses_post(wpautop($content));
2246		- }
2247		- ?>
2248		- </div>
2249		- </div>
2250		- </div>
2251		- </td>
2252		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2253		- <?php
2254		- $actual_source = $source_url;
2255		- if (strpos($source_url, '_ungrouped_') === 0) {
2256		- $actual_source = $prompt->source_url ?? '';
2257		- }
2258		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
2259		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2260		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2261		- <?php esc_html_e('View', 'mxchat'); ?>
2262		- </a>
2263		- <?php else : ?>
2264		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
2265		- <?php endif; ?>
2266		- </td>
2267		- <td style="padding: 12px 16px;">
2268		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-ajax" data-vector-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_pinecone_prompt_nonce'); ?>" style="color: var(--mxch-error);">
2269		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2270		- </button>
2271		- </td>
2272		- </tr>
2273		- <?php
2274		- }
2275		- }
2276		- }
2277		- $html = ob_get_clean();
2278		-
2279		- // Generate pagination HTML for Pinecone
2280		- $total_pages = ceil($total_records / $per_page);
2281		- $pagination_html = '';
2282		- if ($total_pages > 1) {
2283		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
2284		-
2285		- // Previous button
2286		- if ($page > 1) {
2287		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
2288		- }
2289		-
2290		- // Page numbers
2291		- $start_page = max(1, $page - 2);
2292		- $end_page = min($total_pages, $page + 2);
2293		-
2294		- if ($start_page > 1) {
2295		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
2296		- if ($start_page > 2) {
2297		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2298		- }
2299		- }
2300		-
2301		- for ($i = $start_page; $i <= $end_page; $i++) {
2302		- if ($i == $page) {
2303		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
2304		- } else {
2305		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
2306		- }
2307		- }
2308		-
2309		- if ($end_page < $total_pages) {
2310		- if ($end_page < $total_pages - 1) {
2311		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2312		- }
2313		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
2314		- }
2315		-
2316		- // Next button
2317		- if ($page < $total_pages) {
2318		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
2319		- }
2320		-
2321		- $pagination_html .= '</div>';
2322		- }
2323		-
2324		- wp_send_json_success(array(
2325		- 'html' => $html,
2326		- 'pagination_html' => $pagination_html,
2327		- 'total_count' => $total_records,
2328		- 'total_pages' => $total_pages,
2329		- 'page' => $page,
2330		- 'per_page' => $per_page,
2331		- 'data_source' => 'pinecone'
2332		- ));
2333		-}
2334		-
2335		-/**
2336		- * AJAX handler for pagination - handles both WordPress DB and Pinecone data sources
2337		- * Returns paginated entries without requiring a full page reload
2338		- */
2339		-public function ajax_mxchat_paginate_entries() {
2340		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2341		-
2342		- if (!current_user_can('manage_options')) {
2343		- wp_send_json_error(array('message' => 'Unauthorized'));
2344		- return;
2345		- }
2346		-
2347		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2348		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2349		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2350		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2351		- $per_page = 25;
2352		-
2353		- // Check if Pinecone is enabled for this bot
2354		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
2355		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
2356		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2357		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
2358		-
2359		- if ($use_pinecone && $has_pinecone_api) {
2360		- // Delegate to Pinecone pagination handler (pass search params)
2361		- $_POST['page'] = $page;
2362		- $_POST['search'] = $search_query;
2363		- $_POST['content_type'] = $content_type_filter;
2364		- $this->ajax_mxchat_refresh_pinecone_entries();
2365		- return;
2366		- }
2367		-
2368		- // WordPress DB pagination - MUST match initial page load logic exactly
2369		- global $wpdb;
2370		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2371		- $offset = ($page - 1) * $per_page;
2372		-
2373		- // Build WHERE clause for search and content type filtering
2374		- $where_clauses = array();
2375		- $where_values = array();
2376		-
2377		- if ($search_query) {
2378		- $where_clauses[] = "article_content LIKE %s";
2379		- $where_values[] = '%' . $wpdb->esc_like($search_query) . '%';
2380		- }
2381		-
2382		- if ($content_type_filter) {
2383		- switch ($content_type_filter) {
2384		- case 'manual':
2385		- $where_clauses[] = "(source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')";
2386		- break;
2387		- case 'pdf':
2388		- $where_clauses[] = "source_url LIKE '%.pdf'";
2389		- break;
2390		- case 'url':
2391		- $where_clauses[] = "source_url != '' AND source_url IS NOT NULL AND source_url NOT LIKE 'mxchat://%' AND source_url NOT LIKE '%.pdf'";
2392		- break;
2393		- }
2394		- }
2395		-
2396		- $where_sql = !empty($where_clauses) ? 'WHERE ' . implode(' AND ', $where_clauses) : '';
2397		-
2398		- // Count grouped entries with filters applied
2399		- if (!empty($where_values)) {
2400		- $count_args = array_merge($where_values, $where_values);
2401		- $count_query = $wpdb->prepare(
2402		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2403		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))",
2404		- ...$count_args
2405		- );
2406		- $total_records = $wpdb->get_var($count_query);
2407		- } else if (!empty($where_sql)) {
2408		- // Content type filter only (no search), no prepared values needed
2409		- $total_records = $wpdb->get_var(
2410		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2411		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))"
2412		- );
2413		- } else {
2414		- // No filters
2415		- $total_records = $wpdb->get_var(
2416		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2417		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
2418		- );
2419		- }
2420		- $total_pages = ceil($total_records / $per_page);
2421		-
2422		- // Step 1: Get unique source_urls for this page (ordered by latest timestamp) with filters
2423		- if (!empty($where_values)) {
2424		- $query_args = array_merge($where_values, array($per_page, $offset));
2425		- $urls_query = $wpdb->prepare(
2426		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2427		- {$where_sql}
2428		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2429		- ...$query_args
2430		- );
2431		- } else if (!empty($where_sql)) {
2432		- $urls_query = $wpdb->prepare(
2433		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2434		- {$where_sql}
2435		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2436		- $per_page, $offset
2437		- );
2438		- } else {
2439		- $urls_query = $wpdb->prepare(
2440		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2441		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2442		- $per_page, $offset
2443		- );
2444		- }
2445		- $page_urls = $wpdb->get_results($urls_query);
2446		-
2447		- // Step 2: Build list of source_urls to fetch
2448		- $url_list = array();
2449		- $url_order_map = array();
2450		- $order_index = 0;
2451		- foreach ($page_urls as $url_row) {
2452		- $url = $url_row->source_url;
2453		- $url_list[] = $url;
2454		- $url_order_map[$url] = $order_index++;
2455		- }
2456		-
2457		- // Step 3: Fetch all rows for these source_urls (with search filter if applicable)
2458		- $prompts = array();
2459		- if (!empty($url_list)) {
2460		- $placeholders = implode(',', array_fill(0, count($url_list), '%s'));
2461		- if ($search_query) {
2462		- // Include search filter in the final fetch
2463		- $prompts_query = $wpdb->prepare(
2464		- "SELECT id, article_content, source_url, timestamp, role_restriction
2465		- FROM {$table_name}
2466		- WHERE source_url IN ($placeholders) AND article_content LIKE %s
2467		- ORDER BY timestamp DESC",
2468		- ...array_merge($url_list, ['%' . $wpdb->esc_like($search_query) . '%'])
2469		- );
2470		- } else {
2471		- $prompts_query = $wpdb->prepare(
2472		- "SELECT id, article_content, source_url, timestamp, role_restriction
2473		- FROM {$table_name}
2474		- WHERE source_url IN ($placeholders)
2475		- ORDER BY timestamp DESC",
2476		- $url_list
2477		- );
2478		- }
2479		- $prompts = $wpdb->get_results($prompts_query);
2480		- }
2481		-
2482		- // Group prompts by source_url for chunk display
2483		- $grouped_prompts = array();
2484		- foreach ($prompts as $prompt) {
2485		- $source_url = $prompt->source_url ?? '';
2486		-
2487		- // Parse chunk metadata using the proper chunker method (same as initial page load)
2488		- if (class_exists('MxChat_Chunker')) {
2489		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($prompt->article_content);
2490		- $prompt->chunk_metadata = $chunk_meta['metadata'];
2491		- $prompt->display_content = $chunk_meta['text'];
2492		- } else {
2493		- $prompt->chunk_metadata = array();
2494		- $prompt->display_content = $prompt->article_content;
2495		- }
2496		-
2497		- if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0) {
2498		- if (!isset($grouped_prompts[$source_url])) {
2499		- $grouped_prompts[$source_url] = array();
2500		- }
2501		- $grouped_prompts[$source_url][] = $prompt;
2502		- } else {
2503		- // Ungrouped entries
2504		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2505		- }
2506		- }
2507		-
2508		- // Sort groups by the original URL order (newest first)
2509		- uksort($grouped_prompts, function($a, $b) use ($url_order_map) {
2510		- $order_a = isset($url_order_map[$a]) ? $url_order_map[$a] : PHP_INT_MAX;
2511		- $order_b = isset($url_order_map[$b]) ? $url_order_map[$b] : PHP_INT_MAX;
2512		- return $order_a - $order_b;
2513		- });
2514		-
2515		- // Sort each group internally by chunk_index
2516		- foreach ($grouped_prompts as $source_url => &$group) {
2517		- usort($group, function($a, $b) {
2518		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2519		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2520		- return $index_a - $index_b;
2521		- });
2522		- }
2523		- unset($group);
2524		-
2525		- // Build HTML for the table rows
2526		- ob_start();
2527		- $display_index = 0;
2528		- $current_page = $page;
2529		- $data_source = 'wordpress';
2530		- $current_bot_id = $bot_id;
2531		- $preview_length = 150;
2532		-
2533		- if (empty($grouped_prompts)) {
2534		- echo '<tr><td colspan="5" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2535		- esc_html_e('No knowledge entries found. Use the Import Options to add content.', 'mxchat');
2536		- echo '</td></tr>';
2537		- } else {
2538		- foreach ($grouped_prompts as $source_url => $group) {
2539		- $chunk_count = count($group);
2540		- $first_prompt = $group[0];
2541		- $display_index++;
2542		-
2543		- if ($chunk_count > 1) {
2544		- // Multiple chunks - show grouped row with expand button
2545		- $group_id = 'group-' . md5($source_url);
2546		- ?>
2547		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2548		- class="mxchat-chunk-group-header"
2549		- data-source="<?php echo esc_attr($data_source); ?>"
2550		- data-group-id="<?php echo esc_attr($group_id); ?>"
2551		- style="border-bottom: 1px solid var(--mxch-card-border);">
2552		- <td style="padding: 12px 16px; text-align: center;">
2553		- <input type="checkbox"
2554		- class="mxchat-entry-checkbox"
2555		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2556		- data-source="<?php echo esc_attr($data_source); ?>"
2557		- data-source-url="<?php echo esc_attr($source_url); ?>"
2558		- data-is-group="true"
2559		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2560		- </td>
2561		- <td style="padding: 12px 16px; font-size: 13px;">
2562		- <?php echo esc_html($first_prompt->id); ?>
2563		- </td>
2564		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2565		- <div class="mxchat-chunk-group-info">
2566		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2567		- <span class="dashicons dashicons-arrow-right-alt2"></span>
2568		- </button>
2569		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2570		- <span class="mxchat-chunk-preview">
2571		- <?php
2572		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : $first_prompt->article_content;
2573		- $content_preview = mb_substr($parent_content, 0, 100);
2574		- echo esc_html($content_preview . '...');
2575		- ?>
2576		- </span>
2577		- </div>
2578		- </td>
2579		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2580		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2581		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2582		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2583		- <?php esc_html_e('View Source', 'mxchat'); ?>
2584		- </a>
2585		- <?php else : ?>
2586		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2587		- <?php endif; ?>
2588		- </td>
2589		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2590		- <?php if ($data_source !== 'pinecone') : ?>
2591		- <button type="button"
2592		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2593		- data-source-url="<?php echo esc_attr($source_url); ?>"
2594		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2595		- data-data-source="<?php echo esc_attr($data_source); ?>"
2596		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2597		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2598		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2599		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2600		- </button>
2601		- <?php endif; ?>
2602		- <button type="button"
2603		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2604		- data-source-url="<?php echo esc_attr($source_url); ?>"
2605		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2606		- data-data-source="<?php echo esc_attr($data_source); ?>"
2607		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2608		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2609		- style="color: var(--mxch-error);"
2610		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2611		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2612		- </button>
2613		- </td>
2614		- </tr>
2615		- <?php
2616		- // Render hidden chunk rows
2617		- foreach ($group as $chunk_index => $chunk) {
2618		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
2619		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
2620		- $content = isset($chunk->display_content) ? $chunk->display_content : $chunk->article_content;
2621		- $content_preview = mb_strlen($content) > $preview_length
2622		- ? mb_substr($content, 0, $preview_length) . '...'
2623		- : $content;
2624		- ?>
2625		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
2626		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
2627		- data-source="<?php echo esc_attr($data_source); ?>"
2628		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
2629		- <td style="padding: 12px 16px; text-align: center;">
2630		- <!-- Checkbox column placeholder for chunks (managed by group) -->
2631		- </td>
2632		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
2633		- <!-- Hidden ID column for chunks -->
2634		- </td>
2635		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2636		- <div class="mxchat-accordion-wrapper">
2637		- <div class="mxchat-content-preview">
2638		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
2639		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
2640		- </span>
2641		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2642		- <?php if (mb_strlen($content) > $preview_length) : ?>
2643		- <button class="mxchat-expand-toggle" type="button">
2644		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2645		- </button>
2646		- <?php endif; ?>
2647		- </div>
2648		- <div class="mxchat-content-full" style="display: none;">
2649		- <div class="content-view">
2650		- <?php
2651		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2652		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2653		- echo wp_kses_post(wpautop($content));
2654		- echo '</div>';
2655		- } else {
2656		- echo wp_kses_post(wpautop($content));
2657		- }
2658		- ?>
2659		- </div>
2660		- </div>
2661		- </div>
2662		- </td>
2663		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2664		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
2665		- </td>
2666		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
2667		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
2668		- </td>
2669		- </tr>
2670		- <?php
2671		- }
2672		- } else {
2673		- // Single entry - display normally with accordion
2674		- $prompt = $first_prompt;
2675		- $content = isset($prompt->display_content) ? $prompt->display_content : $prompt->article_content;
2676		- $content_preview = mb_strlen($content) > $preview_length
2677		- ? mb_substr($content, 0, $preview_length) . '...'
2678		- : $content;
2679		- ?>
2680		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
2681		- data-source="<?php echo esc_attr($data_source); ?>"
2682		- style="border-bottom: 1px solid var(--mxch-card-border);">
2683		- <td style="padding: 12px 16px; text-align: center;">
2684		- <input type="checkbox"
2685		- class="mxchat-entry-checkbox"
2686		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2687		- data-source="<?php echo esc_attr($data_source); ?>"
2688		- data-source-url="<?php echo esc_attr($source_url); ?>"
2689		- data-is-group="false">
2690		- </td>
2691		- <td style="padding: 12px 16px; font-size: 13px;">
2692		- <?php echo esc_html($prompt->id); ?>
2693		- </td>
2694		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2695		- <div class="mxchat-accordion-wrapper">
2696		- <div class="mxchat-content-preview">
2697		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2698		- <?php if (mb_strlen($content) > $preview_length) : ?>
2699		- <button class="mxchat-expand-toggle" type="button">
2700		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2701		- </button>
2702		- <?php endif; ?>
2703		- </div>
2704		- <div class="mxchat-content-full" style="display: none;">
2705		- <div class="content-view">
2706		- <?php
2707		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2708		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2709		- echo wp_kses_post(wpautop($content));
2710		- echo '</div>';
2711		- } else {
2712		- echo wp_kses_post(wpautop($content));
2713		- }
2714		- ?>
2715		- </div>
2716		- </div>
2717		- </div>
2718		- </td>
2719		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2720		- <?php
2721		- $actual_source = $source_url;
2722		- if (strpos($source_url, '_ungrouped_') === 0) {
2723		- $actual_source = $prompt->source_url ?? '';
2724		- }
2725		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
2726		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2727		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2728		- <?php esc_html_e('View', 'mxchat'); ?>
2729		- </a>
2730		- <?php else : ?>
2731		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
2732		- <?php endif; ?>
2733		- </td>
2734		- <td style="padding: 12px 16px; white-space: nowrap;">
2735		- <button type="button"
2736		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2737		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
2738		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2739		- data-data-source="<?php echo esc_attr($data_source); ?>"
2740		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2741		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2742		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2743		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2744		- </button>
2745		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-wordpress" data-entry-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_wordpress_prompt_nonce'); ?>" style="color: var(--mxch-error);">
2746		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2747		- </button>
2748		- </td>
2749		- </tr>
2750		- <?php
2751		- }
2752		- }
2753		- }
2754		- $html = ob_get_clean();
2755		-
2756		- // Generate pagination HTML (include search/filter data for subsequent pages)
2757		- $pagination_html = '';
2758		- if ($total_pages > 1) {
2759		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
2760		-
2761		- // Previous button
2762		- if ($page > 1) {
2763		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
2764		- }
2765		-
2766		- // Page numbers
2767		- $start_page = max(1, $page - 2);
2768		- $end_page = min($total_pages, $page + 2);
2769		-
2770		- if ($start_page > 1) {
2771		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
2772		- if ($start_page > 2) {
2773		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2774		- }
2775		- }
2776		-
2777		- for ($i = $start_page; $i <= $end_page; $i++) {
2778		- if ($i == $page) {
2779		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
2780		- } else {
2781		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
2782		- }
2783		- }
2784		-
2785		- if ($end_page < $total_pages) {
2786		- if ($end_page < $total_pages - 1) {
2787		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2788		- }
2789		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
2790		- }
2791		-
2792		- // Next button
2793		- if ($page < $total_pages) {
2794		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
2795		- }
2796		-
2797		- $pagination_html .= '</div>';
2798		- }
2799		-
2800		- wp_send_json_success(array(
2801		- 'html' => $html,
2802		- 'pagination_html' => $pagination_html,
2803		- 'total_count' => $total_records,
2804		- 'total_pages' => $total_pages,
2805		- 'page' => $page,
2806		- 'per_page' => $per_page,
2807		- 'data_source' => 'wordpress'
2808		- ));
2809		-}
2810		-
2811		-/**
2812		- * AJAX handler to detect available sitemaps on the site
2813		- * Optimized for speed - only checks primary sitemap indexes first
2814		- */
2815		-public function ajax_mxchat_detect_sitemaps() {
2816		- check_ajax_referer('mxchat_detect_sitemaps_nonce', 'nonce');
2817		-
2818		- if (!current_user_can('manage_options')) {
2819		- wp_send_json_error(array('message' => 'Unauthorized'));
2820		- return;
2821		- }
2822		-
2823		- $site_url = get_site_url();
2824		- $sitemaps = array();
2825		- $found_index = false;
2826		-
2827		- // Only check the main sitemap index files first (much faster)
2828		- // These are the primary entry points that contain sub-sitemaps
2829		- $primary_indexes = array(
2830		- 'sitemap_index.xml' => 'Yoast SEO / Rank Math', // Yoast & Rank Math
2831		- 'wp-sitemap.xml' => 'WordPress Core', // WordPress Core
2832		- 'sitemap.xml' => 'Standard', // Generic/AIOSEO
2833		- );
2834		-
2835		- foreach ($primary_indexes as $path => $source) {
2836		- $url = trailingslashit($site_url) . $path;
2837		-
2838		- $response = wp_remote_head($url, array(
2839		- 'timeout' => 10,
2840		- 'sslverify' => false,
2841		- 'redirection' => 1,
2842		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2843		- ));
2844		-
2845		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2846		- // Found a sitemap index - parse it to get sub-sitemaps
2847		- $sub_sitemaps = $this->parse_sitemap_index($url);
2848		- if (!empty($sub_sitemaps)) {
2849		- $sitemaps[] = array(
2850		- 'url' => $url,
2851		- 'type' => 'index',
2852		- 'source' => $source,
2853		- 'sub_sitemaps' => $sub_sitemaps
2854		- );
2855		- $found_index = true;
2856		- // Found a valid index, no need to check others
2857		- break;
2858		- }
2859		- }
2860		- }
2861		-
2862		- // If no sitemap index found, check for standalone sitemaps
2863		- if (!$found_index) {
2864		- $standalone_sitemaps = array(
2865		- 'post-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
2866		- 'page-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
2867		- );
2868		-
2869		- foreach ($standalone_sitemaps as $path => $info) {
2870		- $url = trailingslashit($site_url) . $path;
2871		-
2872		- $response = wp_remote_head($url, array(
2873		- 'timeout' => 2,
2874		- 'sslverify' => false
2875		- ));
2876		-
2877		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2878		- $sitemaps[] = array(
2879		- 'url' => $url,
2880		- 'type' => $info['type'],
2881		- 'source' => $info['source'],
2882		- 'url_count' => 0 // Skip URL count for speed
2883		- );
2884		- }
2885		- }
2886		- }
2887		-
2888		- wp_send_json_success(array(
2889		- 'sitemaps' => $sitemaps,
2890		- 'site_url' => $site_url
2891		- ));
2892		-}
2893		-
2894		-/**
2895		- * Parse a sitemap index to get sub-sitemaps
2896		- * Optimized: doesn't fetch URL count for each sub-sitemap (too slow)
2897		- */
2898		-private function parse_sitemap_index($url) {
2899		- $sub_sitemaps = array();
2900		-
2901		- $response = wp_remote_get($url, array(
2902		- 'timeout' => 30,
2903		- 'sslverify' => false,
2904		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2905		- 'headers' => array(
2906		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
2907		- 'Accept-Language' => 'en-US,en;q=0.9',
2908		- ),
2909		- ));
2910		-
2911		- if (is_wp_error($response)) {
2912		- return $sub_sitemaps;
2913		- }
2914		-
2915		- $body = wp_remote_retrieve_body($response);
2916		- if (empty($body)) {
2917		- return $sub_sitemaps;
2918		- }
2919		-
2920		- // Suppress XML errors
2921		- libxml_use_internal_errors(true);
2922		- $xml = simplexml_load_string($body);
2923		- libxml_clear_errors();
2924		-
2925		- if ($xml === false) {
2926		- return $sub_sitemaps;
2927		- }
2928		-
2929		- // Check if it's a sitemap index (contains <sitemap> elements)
2930		- if (isset($xml->sitemap)) {
2931		- foreach ($xml->sitemap as $sitemap) {
2932		- $loc = (string) $sitemap->loc;
2933		- if (!empty($loc)) {
2934		- // Try to determine the type from the URL
2935		- $type = 'content';
2936		- if (strpos($loc, 'category') !== false || strpos($loc, 'tag') !== false || strpos($loc, 'taxonomy') !== false) {
2937		- $type = 'taxonomy';
2938		- } elseif (strpos($loc, 'author') !== false || strpos($loc, 'user') !== false) {
2939		- $type = 'author';
2940		- }
2941		-
2942		- // Skip URL count - too slow to fetch for each sitemap
2943		- $sub_sitemaps[] = array(
2944		- 'url' => $loc,
2945		- 'type' => $type,
2946		- 'url_count' => 0, // Don't fetch - takes too long
2947		- 'name' => basename(parse_url($loc, PHP_URL_PATH))
2948		- );
2949		- }
2950		- }
2951		- }
2952		-
2953		- return $sub_sitemaps;
2954		-}
2955		-
2956		-/**
2957		- * Get URL count from a sitemap
2958		- */
2959		-private function get_sitemap_url_count($url) {
2960		- $response = wp_remote_get($url, array(
2961		- 'timeout' => 30,
2962		- 'sslverify' => false,
2963		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2964		- 'headers' => array(
2965		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
2966		- 'Accept-Language' => 'en-US,en;q=0.9',
2967		- ),
2968		- ));
2969		-
2970		- if (is_wp_error($response)) {
2971		- return 0;
2972		- }
2973		-
2974		- $body = wp_remote_retrieve_body($response);
2975		- if (empty($body)) {
2976		- return 0;
2977		- }
2978		-
2979		- // Count <url> or <loc> elements
2980		- $count = preg_match_all('/<url>/i', $body, $matches);
2981		- return $count ?: 0;
2982		-}
2983		-
2984		-/**
2985		- * Get sitemaps declared in robots.txt
2986		- */
2987		-private function get_sitemaps_from_robots($site_url) {
2988		- $sitemaps = array();
2989		- $robots_url = trailingslashit($site_url) . 'robots.txt';
2990		-
2991		- $response = wp_remote_get($robots_url, array(
2992		- 'timeout' => 15,
2993		- 'sslverify' => false,
2994		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2995		- ));
2996		-
2997		- if (is_wp_error($response)) {
2998		- return $sitemaps;
2999		- }
3000		-
3001		- $body = wp_remote_retrieve_body($response);
3002		- if (empty($body)) {
3003		- return $sitemaps;
3004		- }
3005		-
3006		- // Find Sitemap: declarations
3007		- if (preg_match_all('/^Sitemap:\s*(.+)$/mi', $body, $matches)) {
3008		- foreach ($matches[1] as $sitemap_url) {
3009		- $sitemap_url = trim($sitemap_url);
3010		- if (filter_var($sitemap_url, FILTER_VALIDATE_URL)) {
3011		- $sitemaps[] = $sitemap_url;
3012		- }
3013		- }
3014		- }
3015		-
3016		- return $sitemaps;
3017		-}
3018		-
3019	1767	public function mxchat_stop_processing() {
3020	1768	// Verify permissions
3021	1769	if (!current_user_can('manage_options')) {
3022	1770	wp_die(esc_html__('Unauthorized access', 'mxchat'));
		@@ -3024,52 +1772,28 @@
3024	1772
3025	1773	// Verify nonce
3026	1774	check_admin_referer('mxchat_stop_processing_action', 'mxchat_stop_processing_nonce');
3027	1775
3028		- global $wpdb;
3029		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
3030		-
3031		- // Get active queue IDs
3032		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
3033		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
3034		-
3035		- // Delete all pending items from active queues
3036		- if ($sitemap_queue_id) {
3037		- $wpdb->delete(
3038		- $table_name,
3039		- array(
3040		- 'queue_id' => $sitemap_queue_id,
3041		- 'status' => 'pending'
3042		- ),
3043		- array('%s', '%s')
3044		- );
3045		-
3046		- delete_transient('mxchat_active_queue_sitemap');
	1776	+ // Get the last sitemap URL and clear its transient
	1777	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
	1778	+ if ($sitemap_url) {
	1779	+ delete_transient('mxchat_sitemap_status_' . md5($sitemap_url));
3047	1780	delete_transient('mxchat_last_sitemap_url');
3048	1781	}
3049		-
3050		- if ($pdf_queue_id) {
3051		- // Get PDF path before deleting
3052		- $pdf_path = $this->mxchat_get_queue_meta($pdf_queue_id, 'pdf_path');
3053		-
3054		- $wpdb->delete(
3055		- $table_name,
3056		- array(
3057		- 'queue_id' => $pdf_queue_id,
3058		- 'status' => 'pending'
3059		- ),
3060		- array('%s', '%s')
3061		- );
3062		-
3063		- // Delete PDF file
3064		- if ($pdf_path && file_exists($pdf_path)) {
3065		- wp_delete_file($pdf_path);
3066		- }
3067		-
3068		- delete_transient('mxchat_active_queue_pdf');
	1782	+
	1783	+ // Get the last PDF URL and clear its transient
	1784	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1785	+ if ($pdf_url) {
	1786	+ delete_transient('mxchat_pdf_status_' . md5($pdf_url));
3069	1787	delete_transient('mxchat_last_pdf_url');
3070	1788	}
3071	1789
	1790	+ // Unschedule any pending sitemap events
	1791	+ $timestamp = wp_next_scheduled('mxchat_process_sitemap_urls');
	1792	+ if ($timestamp) {
	1793	+ wp_unschedule_event($timestamp, 'mxchat_process_sitemap_urls');
	1794	+ }
	1795	+
3072	1796	// Redirect back with a success message
3073	1797	set_transient('mxchat_admin_notice_success',
3074	1798	esc_html__('Processing has been stopped successfully.', 'mxchat'),
3075	1799	30
		@@ -3076,12 +1800,8 @@
3076	1800	);
3077	1801	wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
3078	1802	exit;
3079	1803	}
3080		-
3081		-/**
3082		- * Get content list for processing
3083		- */
3084	1804	public function ajax_mxchat_get_content_list() {
3085	1805	// Verify the nonce
3086	1806	check_ajax_referer('mxchat_content_selector_nonce', 'nonce');
3087	1807
		@@ -3089,9 +1809,9 @@
3089	1809	wp_send_json_error(__('Unauthorized access', 'mxchat'));
3090	1810	}
3091	1811
3092	1812	$page = isset($_GET['page']) ? absint($_GET['page']) : 1;
3093		- $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 100;
	1813	+ $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 20;
3094	1814	$search = isset($_GET['search']) ? sanitize_text_field($_GET['search']) : '';
3095	1815	$post_type = isset($_GET['post_type']) ? sanitize_text_field($_GET['post_type']) : 'all';
3096	1816	$post_status = isset($_GET['post_status']) ? sanitize_text_field($_GET['post_status']) : 'publish';
3097	1817	$processed_filter = isset($_GET['processed_filter']) ? sanitize_text_field($_GET['processed_filter']) : 'all';
		@@ -3104,47 +1824,20 @@
3104	1824	'orderby' => 'date',
3105	1825	'order' => 'DESC',
3106	1826	);
3107	1827
3108		- // Handle post types - IMPROVED VERSION
	1828	+ // Handle post types
3109	1829	if ($post_type !== 'all') {
3110	1830	$args['post_type'] = $post_type;
3111	1831	} else {
3112		- // Get all available post types that might contain content
3113		- $all_post_types = array();
	1832	+ // Default to post and page if we can't get post types
	1833	+ $args['post_type'] = array('post', 'page');
3114	1834
3115		- // First get all public post types
3116		- $public_types = get_post_types(array('public' => true), 'names');
3117		- $all_post_types = array_merge($all_post_types, $public_types);
3118		-
3119		- // Add common forum/community post types
3120		- $forum_types = array('topic', 'reply', 'forum', 'wpforo_topic', 'wpforo_post');
3121		- foreach ($forum_types as $forum_type) {
3122		- if (post_type_exists($forum_type)) {
3123		- $all_post_types[] = $forum_type;
3124		- }
	1835	+ // Try to get public post types
	1836	+ $public_types = $this->mxchat_get_public_post_types();
	1837	+ if (is_array($public_types) && !empty($public_types)) {
	1838	+ $args['post_type'] = array_keys($public_types);
3125	1839	}
3126		-
3127		- // Add other commonly used post types
3128		- $common_types = array('product', 'job_listing', 'event', 'portfolio');
3129		- foreach ($common_types as $common_type) {
3130		- if (post_type_exists($common_type)) {
3131		- $all_post_types[] = $common_type;
3132		- }
3133		- }
3134		-
3135		- // Remove duplicates and ensure we have at least some post types
3136		- $all_post_types = array_unique($all_post_types);
3137		-
3138		- if (empty($all_post_types)) {
3139		- // Fallback to basic post types
3140		- $all_post_types = array('post', 'page');
3141		- }
3142		-
3143		- $args['post_type'] = $all_post_types;
3144		-
3145		- // Debug logging to see what post types are being queried
3146		- //error_log('MxChat Debug: Querying post types: ' . implode(', ', $all_post_types));
3147	1840	}
3148	1841
3149	1842	if (!empty($search)) {
3150	1843	$args['s'] = $search;
		@@ -3149,58 +1842,36 @@
3149	1842	if (!empty($search)) {
3150	1843	$args['s'] = $search;
3151	1844	}
3152	1845
3153		- // Get processed data from storage
	1846	+ // ================================
	1847	+ // FIXED: Check only the ACTIVE storage method
	1848	+ // ================================
	1849	+
3154	1850	$processed_data = array();
3155	1851
	1852	+ // Check if Pinecone is enabled
3156	1853	$pinecone_options = get_option('mxchat_pinecone_addon_options', array());
3157	1854	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3158		-
	1855	+
3159	1856	if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3160		- // Get fresh data from Pinecone - no caching
	1857	+ // ONLY check Pinecone if it's enabled
3161	1858	$processed_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
3162	1859	} else {
3163		- // WordPress DB checking with better URL matching for all post types
	1860	+ // ONLY check WordPress DB if Pinecone is not enabled
3164	1861	global $wpdb;
3165	1862	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3166		- $processed_items = $wpdb->get_results("SELECT id, source_url, article_content, timestamp FROM {$table_name}");
3167		-
3168		- // Group items by source_url to count chunks
3169		- $url_chunk_counts = array();
3170		- $url_latest_timestamp = array();
3171		- $url_first_id = array();
3172		-
	1863	+ $processed_items = $wpdb->get_results("SELECT id, source_url, timestamp FROM {$table_name}");
	1864	+
3173	1865	if (!empty($processed_items)) {
3174	1866	foreach ($processed_items as $item) {
3175		- $url = $item->source_url;
3176		- if (empty($url)) continue;
3177		-
3178		- // Count chunks per URL
3179		- if (!isset($url_chunk_counts[$url])) {
3180		- $url_chunk_counts[$url] = 0;
3181		- $url_latest_timestamp[$url] = $item->timestamp;
3182		- $url_first_id[$url] = $item->id;
3183		- }
3184		- $url_chunk_counts[$url]++;
3185		-
3186		- // Track latest timestamp
3187		- if (strtotime($item->timestamp) > strtotime($url_latest_timestamp[$url])) {
3188		- $url_latest_timestamp[$url] = $item->timestamp;
3189		- }
3190		- }
3191		-
3192		- // Now build processed_data with chunk counts
3193		- foreach ($url_chunk_counts as $url => $chunk_count) {
3194		- $post_id = $this->mxchat_url_to_post_id_improved($url);
3195		-
	1867	+ $post_id = url_to_postid($item->source_url);
3196	1868	if ($post_id) {
3197	1869	$processed_data[$post_id] = array(
3198		- 'db_id' => $url_first_id[$url],
3199		- 'timestamp' => $url_latest_timestamp[$url],
3200		- 'url' => $url,
3201		- 'source' => 'wordpress',
3202		- 'chunk_count' => $chunk_count
	1870	+ 'db_id' => $item->id,
	1871	+ 'timestamp' => $item->timestamp,
	1872	+ 'url' => $item->source_url,
	1873	+ 'source' => 'wordpress'
3203	1874	);
3204	1875	}
3205	1876	}
3206	1877	}
		@@ -3205,8 +1876,10 @@
3205	1876	}
3206	1877	}
3207	1878	}
3208	1879
	1880	+ // ================================
	1881	+
3209	1882	// Get processed IDs as a simple array for in_array checks
3210	1883	$processed_ids = array_keys($processed_data);
3211	1884
3212	1885	// Handle processed/unprocessed filter
		@@ -3248,14 +1921,8 @@
3248	1921	$db_record_id = $item_data['db_id'];
3249	1922	}
3250	1923	}
3251	1924
3252		- // Get chunk count for this item
3253		- $chunk_count = 0;
3254		- if ($is_processed && isset($processed_data[$id]['chunk_count'])) {
3255		- $chunk_count = intval($processed_data[$id]['chunk_count']);
3256		- }
3257		-
3258	1925	$content_items[] = array(
3259	1926	'id' => $id,
3260	1927	'title' => get_the_title(),
3261	1928	'permalink' => get_permalink(),
		@@ -3266,10 +1933,9 @@
3266	1933	'word_count' => $word_count,
3267	1934	'already_processed' => $is_processed,
3268	1935	'processed_date' => $processed_date,
3269	1936	'db_record_id' => $db_record_id,
3270		- 'data_source' => $data_source,
3271		- 'chunk_count' => $chunk_count
	1937	+ 'data_source' => $data_source
3272	1938	);
3273	1939	}
3274	1940	wp_reset_postdata();
3275	1941	}
		@@ -3285,178 +1951,8 @@
3285	1951	wp_send_json_success($response);
3286	1952	exit;
3287	1953	}
3288	1954
3289		-
3290		-/**
3291		- * This function handles various WooCommerce URL formats and permalink structures
3292		- */
3293		-private function mxchat_url_to_post_id_improved($url) {
3294		- // First try the standard WordPress function
3295		- $post_id = url_to_postid($url);
3296		-
3297		- if ($post_id > 0) {
3298		- return $post_id;
3299		- }
3300		-
3301		- // If that fails, try more aggressive URL matching
3302		- // Remove trailing slashes and query parameters for better matching
3303		- $clean_url = rtrim($url, '/');
3304		- $clean_url = strtok($clean_url, '?'); // Remove query parameters
3305		-
3306		- // Try again with cleaned URL
3307		- $post_id = url_to_postid($clean_url);
3308		- if ($post_id > 0) {
3309		- return $post_id;
3310		- }
3311		-
3312		- // For bbPress forum topics, try extracting slug from URL
3313		- if (strpos($url, '/topic/') !== false || strpos($url, '/forums/') !== false) {
3314		- // Handle bbPress URLs: /forums/topic/topic-name/
3315		- if (preg_match('/\/forums\/topic\/([^\/\?]+)/', $url, $matches)) {
3316		- $topic_slug = $matches[1];
3317		-
3318		- // Look up topic by slug
3319		- $topic = get_page_by_path($topic_slug, OBJECT, 'topic');
3320		- if ($topic) {
3321		- return $topic->ID;
3322		- }
3323		-
3324		- // Alternative method: query by post_name
3325		- global $wpdb;
3326		- $post_id = $wpdb->get_var($wpdb->prepare(
3327		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3328		- $topic_slug
3329		- ));
3330		-
3331		- if ($post_id) {
3332		- return intval($post_id);
3333		- }
3334		- }
3335		-
3336		- // Handle simpler topic URLs: /topic/topic-name/
3337		- if (preg_match('/\/topic\/([^\/\?]+)/', $url, $matches)) {
3338		- $topic_slug = $matches[1];
3339		-
3340		- global $wpdb;
3341		- $post_id = $wpdb->get_var($wpdb->prepare(
3342		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3343		- $topic_slug
3344		- ));
3345		-
3346		- if ($post_id) {
3347		- return intval($post_id);
3348		- }
3349		- }
3350		- }
3351		-
3352		- // For WooCommerce products
3353		- if (strpos($url, '/product/') !== false || strpos($url, 'product=') !== false) {
3354		- // Extract product slug from various URL formats
3355		- $product_slug = '';
3356		-
3357		- // Handle pretty permalinks: /product/product-name/
3358		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
3359		- $product_slug = $matches[1];
3360		- }
3361		- // Handle query parameters: ?product=product-name
3362		- elseif (preg_match('/[\?&]product=([^&]+)/', $url, $matches)) {
3363		- $product_slug = $matches[1];
3364		- }
3365		-
3366		- if (!empty($product_slug)) {
3367		- // Look up product by slug
3368		- $product = get_page_by_path($product_slug, OBJECT, 'product');
3369		- if ($product) {
3370		- return $product->ID;
3371		- }
3372		-
3373		- // Alternative method: query by post_name
3374		- global $wpdb;
3375		- $post_id = $wpdb->get_var($wpdb->prepare(
3376		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'product' AND post_status = 'publish'",
3377		- $product_slug
3378		- ));
3379		-
3380		- if ($post_id) {
3381		- return intval($post_id);
3382		- }
3383		- }
3384		- }
3385		-
3386		- // Generic approach: try to extract slug and match against all post types
3387		- $parsed_url = wp_parse_url($clean_url);
3388		- $path = $parsed_url['path'] ?? '';
3389		-
3390		- if (!empty($path)) {
3391		- // Get the last part of the path as potential slug
3392		- $path_parts = array_filter(explode('/', trim($path, '/')));
3393		- $potential_slug = end($path_parts);
3394		-
3395		- if (!empty($potential_slug)) {
3396		- global $wpdb;
3397		-
3398		- // Try to find any post with this slug
3399		- $post_id = $wpdb->get_var($wpdb->prepare(
3400		- "SELECT ID FROM {$wpdb->posts}
3401		- WHERE post_name = %s
3402		- AND post_status IN ('publish', 'closed', 'private')
3403		- AND post_type NOT IN ('revision', 'attachment', 'nav_menu_item')
3404		- ORDER BY CASE
3405		- WHEN post_type = 'post' THEN 1
3406		- WHEN post_type = 'page' THEN 2
3407		- WHEN post_type = 'topic' THEN 3
3408		- WHEN post_type = 'product' THEN 4
3409		- ELSE 5
3410		- END
3411		- LIMIT 1",
3412		- $potential_slug
3413		- ));
3414		-
3415		- if ($post_id) {
3416		- return intval($post_id);
3417		- }
3418		- }
3419		- }
3420		-
3421		- // ADDITIONAL: Try direct database lookup by URL variations
3422		- global $wpdb;
3423		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3424		-
3425		- // Try variations of the URL (with/without trailing slash, http/https)
3426		- $url_variations = array(
3427		- $url,
3428		- rtrim($url, '/'),
3429		- $url . '/',
3430		- str_replace('http://', 'https://', $url),
3431		- str_replace('https://', 'http://', $url),
3432		- str_replace('http://', 'https://', rtrim($url, '/')),
3433		- str_replace('https://', 'http://', rtrim($url, '/'))
3434		- );
3435		-
3436		- // Remove duplicates
3437		- $url_variations = array_unique($url_variations);
3438		-
3439		- foreach ($url_variations as $variation) {
3440		- $existing_record = $wpdb->get_row($wpdb->prepare(
3441		- "SELECT id, source_url FROM $table_name WHERE source_url = %s",
3442		- $variation
3443		- ));
3444		-
3445		- if ($existing_record) {
3446		- // Try to get post ID from this stored URL
3447		- $stored_post_id = url_to_postid($existing_record->source_url);
3448		- if ($stored_post_id > 0) {
3449		- return $stored_post_id;
3450		- }
3451		- }
3452		- }
3453		-
3454		- return 0; // No match found
3455		-}
3456		-/**
3457		- * Process selected content via AJAX
3458		- */
3459	1955	public function ajax_mxchat_process_selected_content() {
3460	1956	// Basic request validation
3461	1957	if (!check_ajax_referer('mxchat_content_selector_nonce', 'nonce', false)) {
3462	1958	wp_send_json_error('Invalid nonce');
		@@ -3480,24 +1976,8 @@
3480	1976	wp_send_json_error('No content selected');
3481	1977	exit;
3482	1978	}
3483	1979
3484		- // Get bot_id from request
3485		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
3486		-
3487		- // ACF→PDF extraction is opt-in per import batch. Persist the last-used value so users
3488		- // don't re-check on every batch; the default is OFF for installs that haven't set it.
3489		- $extract_acf_pdfs = !empty($_POST['extract_acf_pdfs']) && $_POST['extract_acf_pdfs'] !== 'false';
3490		- $mxchat_options = get_option('mxchat_options', array());
3491		- if (!is_array($mxchat_options)) {
3492		- $mxchat_options = array();
3493		- }
3494		- $prior_default = !empty($mxchat_options['acf_pdf_extract_default']);
3495		- if ($prior_default !== $extract_acf_pdfs) {
3496		- $mxchat_options['acf_pdf_extract_default'] = $extract_acf_pdfs ? 1 : 0;
3497		- update_option('mxchat_options', $mxchat_options);
3498		- }
3499		-
3500	1980	// Process only ONE post at a time to avoid request size issues
3501	1981	$post_id = reset($post_ids);
3502	1982	$post = get_post($post_id);
3503	1983
		@@ -3504,185 +1984,15 @@
3504	1984	if (!$post) {
3505	1985	wp_send_json_error('Post not found');
3506	1986	exit;
3507	1987	}
3508		-
3509		- // Allow developers to modify post data before processing into knowledge base
3510		- $post = apply_filters('mxchat_before_process_post', $post, $bot_id);
3511		-
3512		- // Get content including title, short description (for WooCommerce), and main content
3513		- $content = $post->post_title . "\n\n";
3514		-
3515		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
3516		- if (!empty($post->post_excerpt)) {
3517		- // Remove shortcode tags but preserve content inside them
3518		- $clean_excerpt = $this->strip_shortcode_tags_preserve_content($post->post_excerpt);
3519		- $content .= "Short Description: " . wp_strip_all_tags($clean_excerpt) . "\n\n";
3520		- }
3521		-
3522		- // Add main content - remove shortcode tags but preserve content inside them
3523		- $clean_content = $this->strip_shortcode_tags_preserve_content($post->post_content);
3524		- $content .= wp_strip_all_tags($clean_content);
3525		-
3526		- // ADD WOOCOMMERCE PRODUCT DATA (pricing, stock, categories, custom tabs)
3527		- if (get_post_type($post_id) === 'product' && class_exists('WooCommerce')) {
3528		- $product = wc_get_product($post_id);
3529		-
3530		- if ($product) {
3531		- // Get pricing information
3532		- $regular_price = $product->get_regular_price();
3533		- $sale_price = $product->get_sale_price();
3534		- $price = $product->get_price();
3535		- $sku = $product->get_sku();
3536		-
3537		- // Get currency symbol
3538		- $currency_symbol = get_woocommerce_currency_symbol();
3539		-
3540		- // Add pricing information
3541		- $content .= "\n";
3542		- if (!empty($regular_price)) {
3543		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
3544		- } elseif (!empty($price)) {
3545		- $content .= "Price: " . $currency_symbol . $price . "\n";
3546		- }
3547		-
3548		- if (!empty($sale_price) && $sale_price !== $regular_price) {
3549		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
3550		- }
3551		-
3552		- // Handle variable products - show price range
3553		- if ($product->is_type('variable')) {
3554		- $min_price = $product->get_variation_price('min');
3555		- $max_price = $product->get_variation_price('max');
3556		- if ($min_price !== $max_price) {
3557		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
3558		- }
3559		- }
3560		-
3561		- if (!empty($sku)) {
3562		- $content .= "SKU: " . $sku . "\n";
3563		- }
3564		-
3565		- // Get product categories
3566		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
3567		- if (!empty($categories) && !is_wp_error($categories)) {
3568		- $content .= "Categories: " . implode(', ', $categories) . "\n";
3569		- }
3570		- }
3571		-
3572		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
3573		- $custom_tabs = get_post_meta($post_id, 'yikes_woo_products_tabs', true);
3574		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
3575		- foreach ($custom_tabs as $tab) {
3576		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
3577		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
3578		-
3579		- if (!empty($tab_title) && !empty($tab_content)) {
3580		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
3581		- }
3582		- }
3583		- }
3584		-
3585		- // Also check for reusable/saved tabs applied to this product
3586		- $applied_saved_tabs = get_post_meta($post_id, 'yikes_woo_reusable_products_tabs_applied', true);
3587		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
3588		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
3589		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
3590		- foreach ($applied_saved_tabs as $saved_tab_id) {
3591		- if (isset($saved_tabs[$saved_tab_id])) {
3592		- $tab = $saved_tabs[$saved_tab_id];
3593		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
3594		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
3595		-
3596		- if (!empty($tab_title) && !empty($tab_content)) {
3597		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
3598		- }
3599		- }
3600		- }
3601		- }
3602		- }
3603		- }
3604		-
3605		- // ADD ACF FIELDS SUPPORT
3606		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
3607		- $pdf_extracted_count = 0;
3608		- if (!empty($acf_fields)) {
3609		- $acf_content_parts = array();
3610		- $pdf_attachment_ids = array();
3611		-
3612		- foreach ($acf_fields as $field_name => $field_value) {
3613		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
3614		-
3615		- if (!empty($formatted_value)) {
3616		- $field_label = ucwords(str_replace('_', ' ', $field_name));
3617		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
3618		- }
3619		-
3620		- // Walk this field's value tree for any PDF attachment references and queue them for extraction.
3621		- // Only when the user opted into ACF→PDF extraction for this batch; otherwise the ACF text
3622		- // still lands in the KB but the heavier PDF parsing is skipped.
3623		- if ($extract_acf_pdfs) {
3624		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
3625		- }
3626		- }
3627		-
3628		- if (!empty($acf_content_parts)) {
3629		- $content .= "\n\n" . implode("\n", $acf_content_parts);
3630		- }
3631		-
3632		- // Extract text from each unique PDF found in ACF fields and append as a labeled section
3633		- if ($extract_acf_pdfs && !empty($pdf_attachment_ids)) {
3634		- $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
3635		- $pdf_sections = array();
3636		- foreach ($pdf_attachment_ids as $att_id) {
3637		- $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
3638		- if (!empty($pdf_text)) {
3639		- $pdf_title = get_the_title($att_id);
3640		- $pdf_url = wp_get_attachment_url($att_id);
3641		- $header = 'PDF Attachment';
3642		- if (!empty($pdf_title)) {
3643		- $header .= ': ' . $pdf_title;
3644		- }
3645		- if (!empty($pdf_url)) {
3646		- $header .= ' (' . $pdf_url . ')';
3647		- }
3648		- $pdf_sections[] = $header . "\n" . $pdf_text;
3649		- $pdf_extracted_count++;
3650		- }
3651		- }
3652		- if (!empty($pdf_sections)) {
3653		- $content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
3654		- }
3655		- }
3656		- }
3657		-
3658		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
3659		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
3660		- if (!empty($custom_meta)) {
3661		- $meta_content_parts = array();
3662		-
3663		- foreach ($custom_meta as $meta_key => $meta_value) {
3664		- // Convert meta key to readable label
3665		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
3666		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
3667		- }
3668		-
3669		- if (!empty($meta_content_parts)) {
3670		- $content .= "\n\n" . implode("\n", $meta_content_parts);
3671		- }
3672		- }
3673		-
3674		- // Debug logging for WordPress Import content
3675		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Post ID: ' . $post_id . ' Title: ' . $post->post_title);
3676		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Raw post_content length: ' . strlen($post->post_content));
3677		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Final content length: ' . strlen($content));
3678		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Content preview: ' . substr($content, 0, 300));
3679		-
3680		- // Note: Removed 10,000 char limit - chunking now handles large content properly
3681		-
3682		- // Get bot-specific API key
3683		- $bot_options = $this->get_bot_options($bot_id);
3684		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	1988	+
	1989	+ // Get minimal content
	1990	+ $content = $post->post_title . "\n\n" . wp_strip_all_tags($post->post_content);
	1991	+ $content = substr($content, 0, 10000); // Limit content size
	1992	+
	1993	+ // Get API key with proper model detection
	1994	+ $options = get_option('mxchat_options');
3685	1995	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
3686	1996
3687	1997	if (strpos($selected_model, 'voyage') === 0) {
3688	1998	$api_key = $options['voyage_api_key'] ?? '';
		@@ -3695,9 +2005,8 @@
3695	2005	$provider_name = 'OpenAI';
3696	2006	}
3697	2007
3698	2008	if (empty($api_key)) {
3699		- MxChat_Admin::mxchat_log_debug('api_error', $provider_name . ' API key not configured for knowledge processing');
3700	2009	wp_send_json_error($provider_name . ' API key not configured');
3701	2010	exit;
3702	2011	}
3703	2012
		@@ -3703,23 +2012,26 @@
3703	2012
3704	2013	$source_url = get_permalink($post_id);
3705	2014	$vector_id = md5($source_url); // Vector ID for Pinecone
3706	2015
3707		- // Check for existing content in bot-specific storage
	2016	+ // ================================
	2017	+ // FIXED: Check for existing content in ONLY the active storage method
	2018	+ // ================================
	2019	+
3708	2020	$is_update = false;
3709	2021
3710		- // Get bot-specific Pinecone configuration
3711		- $bot_pinecone_config = $this->get_bot_pinecone_config($bot_id);
3712		- $use_pinecone = !empty($bot_pinecone_config) && ($bot_pinecone_config['use_pinecone'] ?? false);
	2022	+ // Check if Pinecone is enabled
	2023	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	2024	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3713	2025
3714		- if ($use_pinecone && !empty($bot_pinecone_config['api_key'])) {
3715		- // Check Pinecone for this bot
3716		- $pinecone_data = $this->mxchat_get_pinecone_processed_content($bot_pinecone_config);
	2026	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2027	+ // ONLY check Pinecone if it's enabled
	2028	+ $pinecone_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
3717	2029	if (isset($pinecone_data[$post_id])) {
3718	2030	$is_update = true;
3719	2031	}
3720	2032	} else {
3721		- // Check WordPress DB (same as before since it's shared)
	2033	+ // ONLY check WordPress DB if Pinecone is not enabled
3722	2034	global $wpdb;
3723	2035	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3724	2036	$existing_record = $wpdb->get_row($wpdb->prepare(
3725	2037	"SELECT id FROM $table_name WHERE source_url = %s",
		@@ -3729,54 +2041,54 @@
3729	2041	if ($existing_record) {
3730	2042	$is_update = true;
3731	2043	}
3732	2044	}
3733		-
3734		- // UPDATED 2.5.6: Determine content type based on post_type
3735		- $post_type = $post->post_type;
3736		- $content_type = 'content'; // Default fallback
3737		-
3738		- // Map WordPress post types to content types
3739		- switch ($post_type) {
3740		- case 'post':
3741		- $content_type = 'post';
3742		- break;
3743		- case 'page':
3744		- $content_type = 'page';
3745		- break;
3746		- case 'product':
3747		- $content_type = 'product';
3748		- break;
3749		- default:
3750		- // For custom post types, use the post type name
3751		- $content_type = sanitize_key($post_type);
3752		- break;
3753		- }
3754		-
3755		- // Use the centralized utility function with bot_id and content_type
	2045	+
	2046	+ // Use the centralized utility function for storage
3756	2047	$result = MxChat_Utils::submit_content_to_db(
3757		- $content,
3758		- $source_url,
	2048	+ $content,
	2049	+ $source_url,
3759	2050	$api_key,
3760		- $vector_id,
3761		- $bot_id,
3762		- $content_type
	2051	+ $vector_id
3763	2052	);
3764		-
	2053	+
3765	2054	if (is_wp_error($result)) {
3766		- MxChat_Admin::mxchat_log_debug('storage_error', 'Knowledge storage failed: ' . $result->get_error_message(), array('source_url' => $source_url));
3767	2055	wp_send_json_error('Storage failed: ' . $result->get_error_message());
3768	2056	exit;
3769	2057	}
3770		-
3771		- // Automatically apply role restriction based on tags
3772		- $this->apply_role_restriction_to_post($post_id, $source_url);
3773		-
	2058	+
	2059	+ // ================================
	2060	+ // UPDATE: Only update caches if Pinecone is enabled
	2061	+ // ================================
	2062	+
	2063	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2064	+ // Update vector ID cache for improved fetching
	2065	+ $this->mxchat_update_pinecone_vector_cache($vector_id);
	2066	+
	2067	+ // Update local processed content cache for immediate UI feedback
	2068	+ $pinecone_cache = get_option('mxchat_pinecone_processed_cache', array());
	2069	+ $pinecone_cache[$post_id] = array(
	2070	+ 'db_id' => $vector_id,
	2071	+ 'processed_date' => 'Just now',
	2072	+ 'url' => $source_url,
	2073	+ 'source' => 'pinecone',
	2074	+ 'timestamp' => current_time('timestamp')
	2075	+ );
	2076	+ update_option('mxchat_pinecone_processed_cache', $pinecone_cache);
	2077	+
	2078	+ // Also update the general processed content cache
	2079	+ $processed_cache = get_option('mxchat_processed_content_cache', array());
	2080	+ $processed_cache[$post_id] = array(
	2081	+ 'db_id' => $vector_id,
	2082	+ 'timestamp' => current_time('timestamp'),
	2083	+ 'url' => $source_url,
	2084	+ 'source' => 'pinecone'
	2085	+ );
	2086	+ update_option('mxchat_processed_content_cache', $processed_cache);
	2087	+ }
	2088	+
3774	2089	$operation_type = $is_update ? 'update' : 'new';
3775	2090
3776		- // Count ACF fields for debugging
3777		- $acf_field_count = count($acf_fields);
3778		-
3779	2091	// Success response with minimal data
3780	2092	wp_send_json_success(array(
3781	2093	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
3782	2094	'post_id' => $post_id,
		@@ -3781,95 +2093,27 @@
3781	2093	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
3782	2094	'post_id' => $post_id,
3783	2095	'title' => $post->post_title,
3784	2096	'operation_type' => $operation_type,
3785		- 'vector_id' => $vector_id,
3786		- 'acf_fields_found' => $acf_field_count,
3787		- 'pdf_extracted_count' => (int) $pdf_extracted_count,
3788		- 'content_preview' => substr($content, 0, 100) . '...',
3789		- 'bot_id' => $bot_id
	2097	+ 'vector_id' => $vector_id, // Include vector ID for debugging
	2098	+ 'cache_updated' => $use_pinecone // Indicate if cache was updated
3790	2099	));
3791	2100	exit;
3792	2101	}
3793	2102
3794		-private function apply_role_restriction_to_post($post_id, $source_url) {
3795		- // Get tag-role mappings
3796		- $mappings = get_option('mxchat_tag_role_mappings', array());
3797		-
3798		- if (empty($mappings)) {
3799		- return; // No mappings, leave as public
3800		- }
3801		-
3802		- // Get all tags for the post
3803		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
3804		-
3805		- if (empty($post_tags)) {
3806		- return; // No tags, leave as public
3807		- }
3808		-
3809		- // Determine the highest role restriction based on tags
3810		- $highest_role = 'public';
3811		- $role_hierarchy = array(
3812		- 'public' => 0,
3813		- 'logged_in' => 1,
3814		- 'subscriber' => 2,
3815		- 'contributor' => 3,
3816		- 'author' => 4,
3817		- 'editor' => 5,
3818		- 'administrator' => 6
3819		- );
3820		-
3821		- foreach ($post_tags as $tag_slug) {
3822		- if (isset($mappings[$tag_slug])) {
3823		- $role = $mappings[$tag_slug];
3824		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
3825		- $highest_role = $role;
3826		- }
3827		- }
3828		- }
3829		-
3830		- // If no restricted tags found, return (leave as public)
3831		- if ($highest_role === 'public') {
3832		- return;
3833		- }
3834		-
3835		- // Update the role restriction in the database
3836		- global $wpdb;
3837		-
3838		- // Check if using Pinecone
3839		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
3840		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3841		-
3842		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3843		- // Update Pinecone role restriction
3844		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
3845		- $vector_id = md5($source_url);
3846		-
3847		- $wpdb->replace(
3848		- $roles_table,
3849		- array(
3850		- 'vector_id' => $vector_id,
3851		- 'role_restriction' => $highest_role,
3852		- 'updated_at' => current_time('mysql')
3853		- ),
3854		- array('%s', '%s', '%s')
3855		- );
3856		- } else {
3857		- // Update WordPress DB
3858		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3859		-
3860		- $wpdb->update(
3861		- $table_name,
3862		- array('role_restriction' => $highest_role),
3863		- array('source_url' => $source_url),
3864		- array('%s'),
3865		- array('%s')
3866		- );
3867		- }
3868		-}
3869	2103
	2104	+
	2105	+ /**
	2106	+ * Updates cache with new vector ID if absent
	2107	+ */
	2108	+ public function mxchat_update_pinecone_vector_cache($vector_id) {
	2109	+ $cached_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2110	+ if (!in_array($vector_id, $cached_ids)) {
	2111	+ $cached_ids[] = $vector_id;
	2112	+ update_option('mxchat_pinecone_vector_ids_cache', $cached_ids);
	2113	+ }
	2114	+ }
3870	2115	public function mxchat_get_public_post_types() {
3871		- // Get all public post types
3872	2116	$post_types = get_post_types(array('public' => true), 'objects');
3873	2117	$post_type_options = array();
3874	2118
3875	2119	foreach ($post_types as $post_type) {
		@@ -3875,47 +2119,41 @@
3875	2119	foreach ($post_types as $post_type) {
3876	2120	$post_type_options[$post_type->name] = $post_type->label;
3877	2121	}
3878	2122
3879		- // Also include common forum/community post types that might not be marked as public
3880		- $additional_types = array(
3881		- 'topic' => 'Forum Topics (bbPress)',
3882		- 'reply' => 'Forum Replies (bbPress)',
3883		- 'forum' => 'Forums (bbPress)',
3884		- 'wpforo_topic' => 'wpForo Topics',
3885		- 'wpforo_post' => 'wpForo Posts'
3886		- );
3887		-
3888		- foreach ($additional_types as $type_name => $type_label) {
3889		- if (post_type_exists($type_name) && !isset($post_type_options[$type_name])) {
3890		- $post_type_options[$type_name] = $type_label;
3891		- }
3892		- }
3893		-
3894	2123	return $post_type_options;
3895	2124	}
3896		-
3897		-/**
3898		- * Retrieves processed content from Pinecone API
3899		- */
3900	2125	public function mxchat_get_pinecone_processed_content($pinecone_options) {
	2126	+ // First check local cache for immediate updates
	2127	+ $cached_data = get_option('mxchat_pinecone_processed_cache', array());
	2128	+
3901	2129	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
3902	2130	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
3903		-
	2131	+
3904	2132	if (empty($api_key) || empty($host)) {
3905		- return array();
	2133	+ // Return only cached data if API credentials are missing
	2134	+ return $cached_data;
3906	2135	}
3907		-
	2136	+
3908	2137	$pinecone_data = array();
3909		-
	2138	+
3910	2139	try {
3911		- // Always get fresh data from Pinecone
3912		- $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
3913		-
3914		- // Method 2: Final fallback - try stats endpoint (if available)
	2140	+ // Method 1: Try to get vectors using cached vector IDs first
	2141	+ $cached_vector_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2142	+
	2143	+ if (!empty($cached_vector_ids)) {
	2144	+ $pinecone_data = $this->mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $cached_vector_ids);
	2145	+ }
	2146	+
	2147	+ // Method 2: If no cached IDs or fetch failed, use scanning approach
3915	2148	if (empty($pinecone_data)) {
	2149	+ $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
	2150	+ }
	2151	+
	2152	+ // Method 3: Final fallback - try stats endpoint (if available)
	2153	+ if (empty($pinecone_data)) {
3916	2154	$stats_url = "https://{$host}/describe_index_stats";
3917		-
	2155	+
3918	2156	$response = wp_remote_post($stats_url, array(
3919	2157	'headers' => array(
3920	2158	'Api-Key' => $api_key,
3921	2159	'Content-Type' => 'application/json'
		@@ -3922,42 +2160,59 @@
3922	2160	),
3923	2161	'body' => json_encode(array()),
3924	2162	'timeout' => 30
3925	2163	));
3926		-
	2164	+
3927	2165	if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3928	2166	$body = wp_remote_retrieve_body($response);
3929	2167	$stats_data = json_decode($body, true);
	2168	+
	2169	+ // Log stats for debugging but don't rely on them for vector listing
	2170	+ //error_log('Pinecone index stats: ' . print_r($stats_data, true));
3930	2171	}
3931	2172	}
3932		-
	2173	+
3933	2174	} catch (Exception $e) {
3934		- // Log error but return fresh data only
	2175	+ //error_log('Pinecone processed content exception: ' . $e->getMessage());
3935	2176	}
3936		-
3937		- return $pinecone_data;
	2177	+
	2178	+ // Merge cached data with Pinecone data
	2179	+ // Cache takes priority for recent updates (within last 5 minutes)
	2180	+ $merged_data = $pinecone_data;
	2181	+
	2182	+ foreach ($cached_data as $post_id => $cache_item) {
	2183	+ $cache_timestamp = $cache_item['timestamp'] ?? 0;
	2184	+ $time_diff = current_time('timestamp') - $cache_timestamp;
	2185	+
	2186	+ // If cache item is recent (less than 5 minutes), prioritize it
	2187	+ if ($time_diff < 300) { // 5 minutes = 300 seconds
	2188	+ $merged_data[$post_id] = $cache_item;
	2189	+ } else {
	2190	+ // If not in Pinecone data and cache is old, keep cache but mark as potentially stale
	2191	+ if (!isset($merged_data[$post_id])) {
	2192	+ $merged_data[$post_id] = $cache_item;
	2193	+ }
	2194	+ }
	2195	+ }
	2196	+
	2197	+ return $merged_data;
3938	2198	}
3939	2199	public function mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $vector_ids) {
3940		- //error_log('=== DEBUG: Starting mxchat_fetch_pinecone_vectors_by_ids ===');
3941		-
3942	2200	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
3943	2201	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
3944		-
	2202	+
3945	2203	if (empty($api_key) || empty($host) || empty($vector_ids)) {
3946		- //error_log('DEBUG: Missing parameters for fetch by IDs');
3947	2204	return array();
3948	2205	}
3949		-
	2206	+
3950	2207	try {
3951	2208	$fetch_url = "https://{$host}/vectors/fetch";
3952		- //error_log('DEBUG: Fetch URL: ' . $fetch_url);
3953		- //error_log('DEBUG: Fetching ' . count($vector_ids) . ' vector IDs');
3954		-
	2209	+
3955	2210	// Pinecone fetch API allows fetching specific vectors by ID
3956	2211	$fetch_data = array(
3957	2212	'ids' => array_values($vector_ids)
3958	2213	);
3959		-
	2214	+
3960	2215	$response = wp_remote_post($fetch_url, array(
3961	2216	'headers' => array(
3962	2217	'Api-Key' => $api_key,
3963	2218	'Content-Type' => 'application/json'
		@@ -3964,45 +2219,39 @@
3964	2219	),
3965	2220	'body' => json_encode($fetch_data),
3966	2221	'timeout' => 30
3967	2222	));
3968		-
	2223	+
3969	2224	if (is_wp_error($response)) {
3970		- //error_log('DEBUG: Fetch by IDs WP error: ' . $response->get_error_message());
	2225	+ //error_log('Pinecone fetch by IDs error: ' . $response->get_error_message());
3971	2226	return array();
3972	2227	}
3973		-
	2228	+
3974	2229	$response_code = wp_remote_retrieve_response_code($response);
3975		- //error_log('DEBUG: Fetch response code: ' . $response_code);
3976		-
3977	2230	if ($response_code !== 200) {
3978		- $error_body = wp_remote_retrieve_body($response);
3979		- //error_log('DEBUG: Fetch failed with body: ' . $error_body);
	2231	+ //error_log('Pinecone fetch by IDs failed with code: ' . $response_code);
3980	2232	return array();
3981	2233	}
3982		-
	2234	+
3983	2235	$body = wp_remote_retrieve_body($response);
3984	2236	$data = json_decode($body, true);
3985	2237
3986		- //error_log('DEBUG: Fetch response structure: ' . print_r(array_keys($data), true));
3987		-
3988	2238	if (!isset($data['vectors'])) {
3989		- //error_log('DEBUG: No vectors key in response');
3990	2239	return array();
3991	2240	}
3992		-
	2241	+
3993	2242	$processed_data = array();
3994		-
	2243	+
3995	2244	foreach ($data['vectors'] as $vector_id => $vector_data) {
3996	2245	$metadata = $vector_data['metadata'] ?? array();
3997	2246	$source_url = $metadata['source_url'] ?? '';
3998		-
	2247	+
3999	2248	if (!empty($source_url)) {
4000	2249	$post_id = url_to_postid($source_url);
4001	2250	if ($post_id) {
4002	2251	$created_at = $metadata['created_at'] ?? '';
4003		- $processed_date = 'Recently';
4004		-
	2252	+ $processed_date = 'Recently'; // Default
	2253	+
4005	2254	if (!empty($created_at)) {
4006	2255	$timestamp = is_numeric($created_at) ? $created_at : strtotime($created_at);
4007	2256	if ($timestamp) {
4008	2257	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
		@@ -4007,9 +2256,9 @@
4007	2256	if ($timestamp) {
4008	2257	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
4009	2258	}
4010	2259	}
4011		-
	2260	+
4012	2261	$processed_data[$post_id] = array(
4013	2262	'db_id' => $vector_id,
4014	2263	'processed_date' => $processed_date,
4015	2264	'url' => $source_url,
		@@ -4018,90 +2267,52 @@
4018	2267	);
4019	2268	}
4020	2269	}
4021	2270	}
4022		-
4023		- //error_log('DEBUG: Processed ' . count($processed_data) . ' vectors from fetch');
	2271	+
4024	2272	return $processed_data;
4025		-
	2273	+
4026	2274	} catch (Exception $e) {
4027		- //error_log('DEBUG: Exception in fetch_pinecone_vectors_by_ids: ' . $e->getMessage());
	2275	+ //error_log('Pinecone fetch by IDs exception: ' . $e->getMessage());
4028	2276	return array();
4029	2277	}
4030	2278	}
4031		-
4032		-/**
4033		- * Get embedding dimensions based on the selected model.
4034		- */
4035		-private function mxchat_get_embedding_dimensions() {
4036		- $options = get_option('mxchat_options', array());
4037		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4038		-
4039		- $model_dimensions = array(
4040		- 'text-embedding-ada-002' => 1536,
4041		- 'text-embedding-3-small' => 1536,
4042		- 'text-embedding-3-large' => 3072,
4043		- 'voyage-2' => 1024,
4044		- 'voyage-large-2' => 1536,
4045		- 'voyage-3-large' => 2048,
4046		- 'gemini-embedding-001' => 1536,
4047		- );
4048		-
4049		- if (strpos($selected_model, 'voyage-3-large') === 0) {
4050		- $custom_dimensions = $options['voyage_output_dimension'] ?? 2048;
4051		- return intval($custom_dimensions);
4052		- }
4053		-
4054		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4055		- $custom_dimensions = $options['gemini_output_dimension'] ?? 1536;
4056		- return intval($custom_dimensions);
4057		- }
4058		-
4059		- return $model_dimensions[$selected_model] ?? 1536;
4060		-}
4061		-
4062		-/**
4063		- * Scan Pinecone for processed content
4064		- */
4065	2279	public function mxchat_scan_pinecone_for_processed_content($pinecone_options) {
4066	2280	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4067	2281	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
4068		-
	2282	+
4069	2283	if (empty($api_key) || empty($host)) {
4070	2284	return array();
4071	2285	}
4072		-
	2286	+
4073	2287	try {
4074	2288	// Use multiple random vectors to get better coverage
4075	2289	$all_matches = array();
4076	2290	$seen_ids = array();
4077		-
4078		- // Get correct dimensions for the configured embedding model
4079		- $dimensions = $this->mxchat_get_embedding_dimensions();
4080		-
	2291	+
4081	2292	// Try 3 different random vectors to get better coverage
4082	2293	for ($i = 0; $i < 3; $i++) {
4083	2294	$query_url = "https://{$host}/query";
4084		-
	2295	+
4085	2296	// Generate a random unit vector instead of zeros
4086	2297	$random_vector = array();
4087		- for ($j = 0; $j < $dimensions; $j++) {
4088		- $random_vector[] = (rand(-1000, 1000) / 1000.0);
	2298	+ for ($j = 0; $j < 1536; $j++) {
	2299	+ $random_vector[] = (rand(-1000, 1000) / 1000.0); // Random values between -1 and 1
4089	2300	}
4090		-
	2301	+
4091	2302	// Normalize the vector to unit length
4092	2303	$magnitude = sqrt(array_sum(array_map(function($x) { return $x * $x; }, $random_vector)));
4093	2304	if ($magnitude > 0) {
4094	2305	$random_vector = array_map(function($x) use ($magnitude) { return $x / $magnitude; }, $random_vector);
4095	2306	}
4096		-
	2307	+
4097	2308	$query_data = array(
4098	2309	'includeMetadata' => true,
4099	2310	'includeValues' => false,
4100		- 'topK' => 10000,
	2311	+ 'topK' => 10000, // Get many results
4101	2312	'vector' => $random_vector
4102	2313	);
4103		-
	2314	+
4104	2315	$response = wp_remote_post($query_url, array(
4105	2316	'headers' => array(
4106	2317	'Api-Key' => $api_key,
4107	2318	'Content-Type' => 'application/json'
		@@ -4108,22 +2319,21 @@
4108	2319	),
4109	2320	'body' => json_encode($query_data),
4110	2321	'timeout' => 30
4111	2322	));
4112		-
	2323	+
4113	2324	if (is_wp_error($response)) {
4114	2325	continue;
4115	2326	}
4116		-
	2327	+
4117	2328	$response_code = wp_remote_retrieve_response_code($response);
4118		-
4119	2329	if ($response_code !== 200) {
4120	2330	continue;
4121	2331	}
4122		-
	2332	+
4123	2333	$body = wp_remote_retrieve_body($response);
4124	2334	$data = json_decode($body, true);
4125		-
	2335	+
4126	2336	if (isset($data['matches'])) {
4127	2337	foreach ($data['matches'] as $match) {
4128	2338	$match_id = $match['id'] ?? '';
4129	2339	if (!empty($match_id) && !isset($seen_ids[$match_id])) {
		@@ -4132,30 +2342,24 @@
4132	2342	}
4133	2343	}
4134	2344	}
4135	2345	}
4136		-
4137		- // Convert matches to processed data format, grouping by URL to count chunks
	2346	+
	2347	+ // Convert matches to processed data format
4138	2348	$processed_data = array();
4139		- $url_chunk_counts = array();
4140		-
	2349	+ $vector_ids_for_cache = array();
	2350	+
4141	2351	foreach ($all_matches as $match) {
4142	2352	$metadata = $match['metadata'] ?? array();
4143	2353	$source_url = $metadata['source_url'] ?? '';
4144	2354	$match_id = $match['id'] ?? '';
4145		-
	2355	+
4146	2356	if (!empty($source_url) && !empty($match_id)) {
4147	2357	$post_id = url_to_postid($source_url);
4148	2358	if ($post_id) {
4149		- // Count chunks per post_id
4150		- if (!isset($url_chunk_counts[$post_id])) {
4151		- $url_chunk_counts[$post_id] = 0;
4152		- }
4153		- $url_chunk_counts[$post_id]++;
4154		-
4155	2359	$created_at = $metadata['created_at'] ?? '';
4156		- $processed_date = 'Recently';
4157		-
	2360	+ $processed_date = 'Recently'; // Default
	2361	+
4158	2362	if (!empty($created_at)) {
4159	2363	$timestamp = is_numeric($created_at) ? $created_at : strtotime($created_at);
4160	2364	if ($timestamp) {
4161	2365	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
		@@ -4160,323 +2364,242 @@
4160	2364	if ($timestamp) {
4161	2365	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
4162	2366	}
4163	2367	}
4164		-
4165		- // Only store if not already set, or update with newer timestamp
4166		- if (!isset($processed_data[$post_id]) ||
4167		- ($timestamp ?? 0) > ($processed_data[$post_id]['timestamp'] ?? 0)) {
4168		- $processed_data[$post_id] = array(
4169		- 'db_id' => $match_id,
4170		- 'processed_date' => $processed_date,
4171		- 'url' => $source_url,
4172		- 'source' => 'pinecone',
4173		- 'timestamp' => $timestamp ?? current_time('timestamp')
4174		- );
4175		- }
	2368	+
	2369	+ $processed_data[$post_id] = array(
	2370	+ 'db_id' => $match_id,
	2371	+ 'processed_date' => $processed_date,
	2372	+ 'url' => $source_url,
	2373	+ 'source' => 'pinecone',
	2374	+ 'timestamp' => $timestamp ?? current_time('timestamp')
	2375	+ );
	2376	+
	2377	+ $vector_ids_for_cache[] = $match_id;
4176	2378	}
4177	2379	}
4178	2380	}
4179		-
4180		- // Add chunk counts to processed data
4181		- foreach ($url_chunk_counts as $post_id => $chunk_count) {
4182		- if (isset($processed_data[$post_id])) {
4183		- $processed_data[$post_id]['chunk_count'] = $chunk_count;
4184		- }
	2381	+
	2382	+ // Update the vector IDs cache for future use
	2383	+ if (!empty($vector_ids_for_cache)) {
	2384	+ update_option('mxchat_pinecone_vector_ids_cache', $vector_ids_for_cache);
4185	2385	}
4186		-
	2386	+
4187	2387	return $processed_data;
4188		-
	2388	+
4189	2389	} catch (Exception $e) {
	2390	+ //error_log('Pinecone scan exception: ' . $e->getMessage());
4190	2391	return array();
4191	2392	}
4192		-}
4193		-/**
4194		- * Generate embeddings from input text for MXChat with bot support
4195		- */
4196		-private function mxchat_generate_embedding($text, $bot_id = 'default') {
4197		- // Enable detailed logging for debugging
4198		- //error_log('[MXCHAT-EMBED] Starting embedding generation for bot: ' . $bot_id . '. Text length: ' . strlen($text) . ' bytes');
4199		- //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
	2393	+}
4200	2394
4201		- // Get bot-specific options
4202		- $bot_options = $this->get_bot_options($bot_id);
4203		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	2395	+ /**
	2396	+ * Generates embeddings from input text for MXChat
	2397	+ */
	2398	+ private function mxchat_generate_embedding($text) {
	2399	+ // Enable detailed logging for debugging
	2400	+ //error_log('[MXCHAT-EMBED] Starting embedding generation. Text length: ' . strlen($text) . ' bytes');
	2401	+ //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4204	2402
4205		- // Opt-in: when the custom provider is selected for embeddings, index through
4206		- // the same custom endpoint the query path uses so stored vectors and query
4207		- // vectors share a model. Returns the vector array on success, or an error
4208		- // string on failure (this function's existing failure contract).
4209		- if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
4210		- if (!class_exists('MxChat_Utils')) {
4211		- require_once dirname(__FILE__) . '/../includes/class-mxchat-utils.php';
4212		- }
4213		- return MxChat_Utils::generate_embedding_custom($text, $options);
4214		- }
	2403	+ $options = get_option('mxchat_options');
	2404	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	2405	+ //error_log('[MXCHAT-EMBED] Selected embedding model: ' . $selected_model);
4215	2406
4216		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4217		- //error_log('[MXCHAT-EMBED] Selected embedding model for bot ' . $bot_id . ': ' . $selected_model);
	2407	+ // Determine provider and endpoint
	2408	+ if (strpos($selected_model, 'voyage') === 0) {
	2409	+ $api_key = $options['voyage_api_key'] ?? '';
	2410	+ $endpoint = 'https://api.voyageai.com/v1/embeddings';
	2411	+ $provider_name = 'Voyage AI';
	2412	+ //error_log('[MXCHAT-EMBED] Using Voyage AI API');
	2413	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	2414	+ $api_key = $options['gemini_api_key'] ?? '';
	2415	+ $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
	2416	+ $provider_name = 'Google Gemini';
	2417	+ //error_log('[MXCHAT-EMBED] Using Google Gemini API');
	2418	+ } else {
	2419	+ $api_key = $options['api_key'] ?? '';
	2420	+ $endpoint = 'https://api.openai.com/v1/embeddings';
	2421	+ $provider_name = 'OpenAI';
	2422	+ //error_log('[MXCHAT-EMBED] Using OpenAI API');
	2423	+ }
4218	2424
4219		- // Determine provider and endpoint
4220		- if (strpos($selected_model, 'voyage') === 0) {
4221		- $api_key = $options['voyage_api_key'] ?? '';
4222		- $endpoint = 'https://api.voyageai.com/v1/embeddings';
4223		- $provider_name = 'Voyage AI';
4224		- //error_log('[MXCHAT-EMBED] Using Voyage AI API for bot ' . $bot_id);
4225		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
4226		- $api_key = $options['gemini_api_key'] ?? '';
4227		- $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
4228		- $provider_name = 'Google Gemini';
4229		- //error_log('[MXCHAT-EMBED] Using Google Gemini API for bot ' . $bot_id);
4230		- } else {
4231		- $api_key = $options['api_key'] ?? '';
4232		- $endpoint = 'https://api.openai.com/v1/embeddings';
4233		- $provider_name = 'OpenAI';
4234		- //error_log('[MXCHAT-EMBED] Using OpenAI API for bot ' . $bot_id);
4235		- }
	2425	+ //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
4236	2426
4237		- //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
	2427	+ if (empty($api_key)) {
	2428	+ $error_message = sprintf('Missing %s API key. Please configure your API key in the MxChat settings.', $provider_name);
	2429	+ //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
	2430	+ return $error_message;
	2431	+ }
4238	2432
4239		- if (empty($api_key)) {
4240		- $error_message = sprintf('Missing %s API key for bot %s. Please configure your API key in the bot settings.', $provider_name, $bot_id);
4241		- //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
4242		- return $error_message;
4243		- }
	2433	+ // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
	2434	+ $estimated_tokens = ceil(str_word_count($text) / 0.75);
	2435	+ //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
4244	2436
4245		- // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
4246		- $estimated_tokens = ceil(str_word_count($text) / 0.75);
4247		- //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
	2437	+ if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
	2438	+ //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
	2439	+ // Consider truncating text here
	2440	+ }
4248	2441
4249		- if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
4250		- //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
4251		- // Consider truncating text here
4252		- }
	2442	+ // Prepare request body based on provider
	2443	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2444	+ // Gemini API format
	2445	+ $request_body = array(
	2446	+ 'model' => 'models/' . $selected_model,
	2447	+ 'content' => array(
	2448	+ 'parts' => array(
	2449	+ array('text' => $text)
	2450	+ )
	2451	+ )
	2452	+ );
4253	2453
4254		- // Prepare request body based on provider
4255		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4256		- // Gemini API format
4257		- $request_body = array(
4258		- 'model' => 'models/' . $selected_model,
4259		- 'content' => array(
4260		- 'parts' => array(
4261		- array('text' => $text)
4262		- )
4263		- )
4264		- );
	2454	+ // Set output dimensionality to 1536 for consistency with other models
	2455	+ $request_body['outputDimensionality'] = 1536;
	2456	+ } else {
	2457	+ // OpenAI/Voyage API format
	2458	+ $request_body = array(
	2459	+ 'model' => $selected_model,
	2460	+ 'input' => $text
	2461	+ );
4265	2462
4266		- // Set output dimensionality to 1536 for consistency with other models
4267		- $request_body['outputDimensionality'] = 1536;
4268		- } else {
4269		- // OpenAI/Voyage API format
4270		- $request_body = array(
4271		- 'model' => $selected_model,
4272		- 'input' => $text
4273		- );
	2463	+ // Add output_dimension for voyage-3-large model
	2464	+ if ($selected_model === 'voyage-3-large') {
	2465	+ $request_body['output_dimension'] = 2048;
	2466	+ }
	2467	+ }
4274	2468
4275		- // Add output_dimension for voyage-3-large model
4276		- if ($selected_model === 'voyage-3-large') {
4277		- $request_body['output_dimension'] = 2048;
4278		- }
4279		- }
	2469	+ //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
4280	2470
4281		- //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
	2471	+ // Prepare headers based on provider
	2472	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2473	+ // Gemini uses API key as query parameter
	2474	+ $endpoint .= '?key=' . $api_key;
	2475	+ $headers = array(
	2476	+ 'Content-Type' => 'application/json'
	2477	+ );
	2478	+ } else {
	2479	+ // OpenAI/Voyage use Bearer token
	2480	+ $headers = array(
	2481	+ 'Authorization' => 'Bearer ' . $api_key,
	2482	+ 'Content-Type' => 'application/json'
	2483	+ );
	2484	+ }
4282	2485
4283		- // Prepare headers based on provider
4284		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4285		- // Gemini uses API key as query parameter
4286		- $endpoint .= '?key=' . $api_key;
4287		- $headers = array(
4288		- 'Content-Type' => 'application/json'
4289		- );
4290		- } else {
4291		- // OpenAI/Voyage use Bearer token
4292		- $headers = array(
4293		- 'Authorization' => 'Bearer ' . $api_key,
4294		- 'Content-Type' => 'application/json'
4295		- );
4296		- }
	2486	+ // Make API request
	2487	+ //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
	2488	+ $response = wp_remote_post($endpoint, array(
	2489	+ 'body' => wp_json_encode($request_body),
	2490	+ 'headers' => $headers,
	2491	+ 'timeout' => 60 // Increased timeout for large inputs
	2492	+ ));
4297	2493
4298		- // Make API request
4299		- //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
4300		- $response = wp_remote_post($endpoint, array(
4301		- 'body' => wp_json_encode($request_body),
4302		- 'headers' => $headers,
4303		- 'timeout' => 60 // Increased timeout for large inputs
4304		- ));
	2494	+ // Handle wp_remote_post errors
	2495	+ if (is_wp_error($response)) {
	2496	+ $error_message = $response->get_error_message();
	2497	+ //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
	2498	+ return 'Connection error: ' . $error_message;
	2499	+ }
4305	2500
4306		- // Handle wp_remote_post errors
4307		- if (is_wp_error($response)) {
4308		- $error_message = $response->get_error_message();
4309		- //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
4310		- return 'Connection error: ' . $error_message;
4311		- }
	2501	+ // Get and check HTTP response code
	2502	+ $http_code = wp_remote_retrieve_response_code($response);
	2503	+ //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
4312	2504
4313		- // Get and check HTTP response code
4314		- $http_code = wp_remote_retrieve_response_code($response);
4315		- //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
	2505	+ if ($http_code !== 200) {
	2506	+ $error_body = wp_remote_retrieve_body($response);
	2507	+ //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
4316	2508
4317		- if ($http_code !== 200) {
4318		- $error_body = wp_remote_retrieve_body($response);
4319		- //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
	2509	+ // Try to parse error for more details
	2510	+ $error_json = json_decode($error_body, true);
	2511	+ if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
	2512	+ $error_type = $error_json['error']['type'] ?? 'unknown';
	2513	+ $error_message = $error_json['error']['message'] ?? 'No message';
	2514	+ //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
	2515	+ //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
4320	2516
4321		- // Try to parse error for more details
4322		- $error_json = json_decode($error_body, true);
4323		- if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
4324		- $error_type = $error_json['error']['type'] ?? 'unknown';
4325		- $error_message = $error_json['error']['message'] ?? 'No message';
4326		- //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
4327		- //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
	2517	+ // Customize error message for common API errors
	2518	+ if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
	2519	+ $error_message = sprintf('Invalid %s API key. Please check your API key in the MxChat settings.', $provider_name);
	2520	+ } elseif ($error_type === 'authentication_error') {
	2521	+ $error_message = sprintf('%s authentication failed. Please verify your API key in the MxChat settings.', $provider_name);
	2522	+ }
4328	2523
4329		- // Customize error message for common API errors
4330		- if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
4331		- $error_message = sprintf('Invalid %s API key for bot %s. Please check your API key in the bot settings.', $provider_name, $bot_id);
4332		- } elseif ($error_type === 'authentication_error') {
4333		- $error_message = sprintf('%s authentication failed for bot %s. Please verify your API key in the bot settings.', $provider_name, $bot_id);
4334		- }
	2524	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2525	+ return $error_message;
	2526	+ }
4335	2527
4336		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4337		- return $error_message;
4338		- }
	2528	+ $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding", $http_code);
	2529	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2530	+ return $error_message;
	2531	+ }
4339	2532
4340		- $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding for bot %s", $http_code, $bot_id);
4341		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4342		- return $error_message;
4343		- }
	2533	+ // Parse response body
	2534	+ $response_body = wp_remote_retrieve_body($response);
	2535	+ //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
4344	2536
4345		- // Parse response body
4346		- $response_body = wp_remote_retrieve_body($response);
4347		- //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
	2537	+ $response_data = json_decode($response_body, true);
4348	2538
4349		- $response_data = json_decode($response_body, true);
	2539	+ if (json_last_error() !== JSON_ERROR_NONE) {
	2540	+ $error = json_last_error_msg();
	2541	+ //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
	2542	+ //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
	2543	+ return "Failed to parse API response: $error";
	2544	+ }
4350	2545
4351		- if (json_last_error() !== JSON_ERROR_NONE) {
4352		- $error = json_last_error_msg();
4353		- //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
4354		- //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
4355		- return "Failed to parse API response: $error";
4356		- }
	2546	+ // Handle different response formats based on provider
	2547	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2548	+ // Gemini API response format
	2549	+ if (isset($response_data['embedding']['values'])) {
	2550	+ $embedding_dimensions = count($response_data['embedding']['values']);
	2551	+ //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
4357	2552
4358		- // Handle different response formats based on provider
4359		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4360		- // Gemini API response format
4361		- if (isset($response_data['embedding']['values'])) {
4362		- $embedding_dimensions = count($response_data['embedding']['values']);
4363		- //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
	2553	+ // Check if embedding dimensions are as expected (should be 1536)
	2554	+ if ($embedding_dimensions !== 1536) {
	2555	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
	2556	+ }
4364	2557
4365		- // Check if embedding dimensions are as expected (should be 1536)
4366		- if ($embedding_dimensions !== 1536) {
4367		- //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
4368		- }
	2558	+ return $response_data['embedding']['values'];
	2559	+ } else {
	2560	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
	2561	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4369	2562
4370		- return $response_data['embedding']['values'];
4371		- } else {
4372		- //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
4373		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2563	+ if (isset($response_data['error'])) {
	2564	+ $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
	2565	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2566	+ return $error_message;
	2567	+ }
4374	2568
4375		- if (isset($response_data['error'])) {
4376		- $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
4377		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4378		- return $error_message;
4379		- }
	2569	+ $error_message = "Invalid Gemini API response format: No embedding found";
	2570	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2571	+ return $error_message;
	2572	+ }
	2573	+ } else {
	2574	+ // OpenAI/Voyage API response format
	2575	+ if (isset($response_data['data'][0]['embedding'])) {
	2576	+ $embedding_dimensions = count($response_data['data'][0]['embedding']);
	2577	+ //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
4380	2578
4381		- $error_message = "Invalid Gemini API response format: No embedding found";
4382		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4383		- return $error_message;
4384		- }
4385		- } else {
4386		- // OpenAI/Voyage API response format
4387		- if (isset($response_data['data'][0]['embedding'])) {
4388		- $embedding_dimensions = count($response_data['data'][0]['embedding']);
4389		- //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
	2579	+ // Check if embedding dimensions are as expected
	2580	+ if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
	2581	+ ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
	2582	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
	2583	+ }
4390	2584
4391		- // Check if embedding dimensions are as expected
4392		- if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
4393		- ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
4394		- //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
4395		- }
	2585	+ return $response_data['data'][0]['embedding'];
	2586	+ } else {
	2587	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
	2588	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4396	2589
4397		- return $response_data['data'][0]['embedding'];
4398		- } else {
4399		- //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
4400		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2590	+ if (isset($response_data['error'])) {
	2591	+ $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
	2592	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2593	+ return $error_message;
	2594	+ }
4401	2595
4402		- if (isset($response_data['error'])) {
4403		- $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
4404		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4405		- return $error_message;
4406		- }
4407		-
4408		- $error_message = "Invalid API response format: No embedding found";
4409		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4410		- return $error_message;
4411		- }
4412		- }
4413		-}
4414		-
4415		-/**
4416		- * Get bot-specific options for multi-bot functionality
4417		- * Falls back to default options if bot_id is 'default' or multi-bot add-on is not active
4418		- */
4419		-private function get_bot_options($bot_id = 'default') {
4420		- //error_log("MXCHAT DEBUG: get_bot_options called for bot: " . $bot_id);
4421		-
4422		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4423		- //error_log("MXCHAT DEBUG: Using default options (no multi-bot or bot is 'default')");
4424		- return array();
4425		- }
4426		-
4427		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
4428		-
4429		- if (!empty($bot_options)) {
4430		- //error_log("MXCHAT DEBUG: Got bot-specific options from filter");
4431		- if (isset($bot_options['similarity_threshold'])) {
4432		- //error_log(" - similarity_threshold: " . $bot_options['similarity_threshold']);
4433		- }
4434		- }
4435		-
4436		- return is_array($bot_options) ? $bot_options : array();
4437		-}
4438		-
4439		-/**
4440		- * Get bot-specific Pinecone configuration
4441		- * Used in the knowledge retrieval functions
4442		- */
4443		-// Also add debugging to your get_bot_pinecone_config function
4444		-private function get_bot_pinecone_config($bot_id = 'default') {
4445		- //error_log("MXCHAT DEBUG: get_bot_pinecone_config called for bot: " . $bot_id);
4446		-
4447		- // If default bot or multi-bot add-on not active, use default Pinecone config
4448		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4449		- //error_log("MXCHAT DEBUG: Using default Pinecone config (no multi-bot or bot is 'default')");
4450		- $addon_options = get_option('mxchat_pinecone_addon_options', array());
4451		- $config = array(
4452		- 'use_pinecone' => (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1'),
4453		- 'api_key' => $addon_options['mxchat_pinecone_api_key'] ?? '',
4454		- 'host' => $addon_options['mxchat_pinecone_host'] ?? '',
4455		- 'namespace' => $addon_options['mxchat_pinecone_namespace'] ?? ''
4456		- );
4457		- //error_log("MXCHAT DEBUG: Default config - use_pinecone: " . ($config['use_pinecone'] ? 'true' : 'false'));
4458		- return $config;
4459		- }
4460		-
4461		- //error_log("MXCHAT DEBUG: Calling filter 'mxchat_get_bot_pinecone_config' for bot: " . $bot_id);
4462		-
4463		- // Hook for multi-bot add-on to provide bot-specific Pinecone config
4464		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
4465		-
4466		- if (!empty($bot_pinecone_config)) {
4467		- //error_log("MXCHAT DEBUG: Got bot-specific config from filter");
4468		- //error_log(" - use_pinecone: " . (isset($bot_pinecone_config['use_pinecone']) ? ($bot_pinecone_config['use_pinecone'] ? 'true' : 'false') : 'not set'));
4469		- //error_log(" - host: " . ($bot_pinecone_config['host'] ?? 'not set'));
4470		- //error_log(" - namespace: " . ($bot_pinecone_config['namespace'] ?? 'not set'));
4471		- } else {
4472		- //error_log("MXCHAT DEBUG: Filter returned empty config!");
4473		- }
4474		-
4475		- return is_array($bot_pinecone_config) ? $bot_pinecone_config : array();
4476		-}
4477		-
4478		-
	2596	+ $error_message = "Invalid API response format: No embedding found";
	2597	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2598	+ return $error_message;
	2599	+ }
	2600	+ }
	2601	+ }
4479	2602	public function mxchat_ajax_dismiss_completed_status() {
4480	2603	try {
4481	2604	// Verify the request
4482	2605	check_ajax_referer('mxchat_status_nonce', 'nonce');
		@@ -4595,10 +2718,10 @@
4595	2718
4596	2719	$html .= '<p><strong>' . esc_html__('Status:', 'mxchat') . '</strong> ' . esc_html(ucfirst($status['status'])) . '</p>';
4597	2720	$html .= '<p><strong>' . esc_html__('Last update:', 'mxchat') . '</strong> ' . esc_html($status['last_update']) . '</p>';
4598	2721
4599		- // Add completion summary if available AND it's an array
4600		- if (isset($status['completion_summary']) && is_array($status['completion_summary']) && !empty($status['completion_summary'])) {
	2722	+ // Add completion summary if available
	2723	+ if (isset($status['completion_summary'])) {
4601	2724	$summary = $status['completion_summary'];
4602	2725	$html .= '<div class="mxchat-completion-summary">';
4603	2726	$html .= '<h5>' . esc_html__('Processing Summary', 'mxchat') . '</h5>';
4604	2727	$html .= '<p><strong>' . esc_html__('Total Pages:', 'mxchat') . '</strong> ' . esc_html($summary['total_pages']) . '</p>';
		@@ -4607,10 +2730,10 @@
4607	2730	$html .= '<p><strong>' . esc_html__('Completed:', 'mxchat') . '</strong> ' . esc_html($summary['completion_time']) . '</p>';
4608	2731	$html .= '</div>';
4609	2732	}
4610	2733
4611		- // Add failed pages list if any AND it's an array
4612		- if (isset($status['failed_pages_list']) && is_array($status['failed_pages_list']) && !empty($status['failed_pages_list'])) {
	2734	+ // Add failed pages list if any
	2735	+ if (!empty($status['failed_pages_list'])) {
4613	2736	$html .= $this->mxchat_render_failed_pages_list($status['failed_pages_list']);
4614	2737	}
4615	2738
4616	2739	// Add error message if any
		@@ -4624,8 +2747,9 @@
4624	2747	$html .= '</div>'; // End card
4625	2748
4626	2749	return $html;
4627	2750	}
	2751	+
4628	2752	/**
4629	2753	* Render sitemap status card HTML
4630	2754	*/
4631	2755	private function mxchat_render_sitemap_status_card($status, $sitemap_url) {
		@@ -4681,10 +2805,10 @@
4681	2805
4682	2806	$html .= '<p><strong>' . esc_html__('Status:', 'mxchat') . '</strong> ' . esc_html(ucfirst($status['status'])) . '</p>';
4683	2807	$html .= '<p><strong>' . esc_html__('Last update:', 'mxchat') . '</strong> ' . esc_html($status['last_update']) . '</p>';
4684	2808
4685		- // Add completion summary if available AND it's an array
4686		- if (isset($status['completion_summary']) && is_array($status['completion_summary']) && !empty($status['completion_summary'])) {
	2809	+ // Add completion summary if available
	2810	+ if (isset($status['completion_summary'])) {
4687	2811	$summary = $status['completion_summary'];
4688	2812	$html .= '<div class="mxchat-completion-summary">';
4689	2813	$html .= '<h5>' . esc_html__('Processing Summary', 'mxchat') . '</h5>';
4690	2814	$html .= '<p><strong>' . esc_html__('Total URLs:', 'mxchat') . '</strong> ' . esc_html($summary['total_urls']) . '</p>';
		@@ -4713,16 +2837,13 @@
4713	2837	$html .= '</div>'; // End card
4714	2838
4715	2839	return $html;
4716	2840	}
4717		-
4718		-
4719	2841	/**
4720	2842	* Render failed pages list
4721	2843	*/
4722	2844	private function mxchat_render_failed_pages_list($failed_pages_list) {
4723		- // Validate that $failed_pages_list is an array and not empty
4724		- if (!is_array($failed_pages_list) || empty($failed_pages_list)) {
	2845	+ if (empty($failed_pages_list)) {
4725	2846	return '';
4726	2847	}
4727	2848
4728	2849	$html = '<div class="mxchat-error-notice">';
		@@ -4747,13 +2868,8 @@
4747	2868	return ($b['time'] ?? 0) - ($a['time'] ?? 0);
4748	2869	});
4749	2870
4750	2871	foreach ($sorted_failed_pages as $item) {
4751		- // Ensure $item is an array before accessing its elements
4752		- if (!is_array($item)) {
4753		- continue;
4754		- }
4755		-
4756	2872	$time_ago = isset($item['time']) ? human_time_diff($item['time'], current_time('timestamp')) . ' ' . esc_html__('ago', 'mxchat') : 'Unknown';
4757	2873	$html .= '<tr>';
4758	2874	$html .= '<td>' . esc_html__('Page', 'mxchat') . ' ' . esc_html($item['page'] ?? 'Unknown') . '</td>';
4759	2875	$html .= '<td style="word-break: break-word;">' . esc_html($item['error'] ?? 'Unknown error') . '</td>';
		@@ -4771,10 +2887,9 @@
4771	2887	/**
4772	2888	* Render failed URLs list
4773	2889	*/
4774	2890	private function mxchat_render_failed_urls_list($failed_urls_list) {
4775		- // Validate that $failed_urls_list is an array and not empty
4776		- if (!is_array($failed_urls_list) || empty($failed_urls_list)) {
	2891	+ if (empty($failed_urls_list)) {
4777	2892	return '';
4778	2893	}
4779	2894
4780	2895	$html = '<div class="mxchat-failed-urls-container">';
		@@ -4801,13 +2916,8 @@
4801	2916	// Show up to 50 failed URLs
4802	2917	$display_urls = array_slice($sorted_failed_urls, 0, 50);
4803	2918
4804	2919	foreach ($display_urls as $item) {
4805		- // Ensure $item is an array before accessing its elements
4806		- if (!is_array($item)) {
4807		- continue;
4808		- }
4809		-
4810	2920	$url = $item['url'] ?? '';
4811	2921	$time_ago = isset($item['time']) ? human_time_diff($item['time'], current_time('timestamp')) . ' ' . esc_html__('ago', 'mxchat') : 'Unknown';
4812	2922
4813	2923	// Truncate URL for display
		@@ -4839,564 +2949,18 @@
4839	2949
4840	2950	return $html;
4841	2951	}
4842	2952
4843		-/**
4844		- * Get all ACF fields for a specific post, excluding any fields the user has disabled
4845		- */
4846		-public function mxchat_get_acf_fields_for_post($post_id) {
4847		- if (!function_exists('get_fields')) {
4848		- return array();
4849		- }
4850		-
4851		- $fields = get_fields($post_id);
4852		- if (!$fields || !is_array($fields)) {
4853		- return array();
4854		- }
4855		-
4856		- // Get excluded fields from settings
4857		- $excluded_fields = get_option('mxchat_acf_excluded_fields', array());
4858		- if (!empty($excluded_fields) && is_array($excluded_fields)) {
4859		- foreach ($excluded_fields as $excluded_field) {
4860		- if (isset($fields[$excluded_field])) {
4861		- unset($fields[$excluded_field]);
4862		- }
4863		- }
4864		- }
4865		-
4866		- return $fields;
4867		-}
4868		-
4869		-/**
4870		- * Get all registered ACF field groups and their fields for the settings UI
4871		- */
4872		-public function mxchat_get_all_acf_fields() {
4873		- if (!function_exists('acf_get_field_groups') || !function_exists('acf_get_fields')) {
4874		- return array();
4875		- }
4876		-
4877		- $all_fields = array();
4878		- $field_groups = acf_get_field_groups();
4879		-
4880		- if (!empty($field_groups)) {
4881		- foreach ($field_groups as $group) {
4882		- $group_fields = acf_get_fields($group['key']);
4883		- if (!empty($group_fields)) {
4884		- $all_fields[$group['title']] = array();
4885		- foreach ($group_fields as $field) {
4886		- $all_fields[$group['title']][] = array(
4887		- 'name' => $field['name'],
4888		- 'label' => $field['label'],
4889		- 'type' => $field['type']
4890		- );
4891		- }
4892		- }
4893		- }
4894		- }
4895		-
4896		- return $all_fields;
4897		-}
4898		-
4899		-/**
4900		- * Get whitelisted custom post meta for a given post
4901		- * This allows non-ACF meta fields (like OptionTree, theme meta boxes) to be included in embeddings
4902		- */
4903		-public function mxchat_get_whitelisted_post_meta($post_id) {
4904		- $whitelist = get_option('mxchat_custom_meta_whitelist', '');
4905		-
4906		- if (empty($whitelist)) {
4907		- return array();
4908		- }
4909		-
4910		- // Parse the whitelist - one meta key per line
4911		- $meta_keys = array_filter(array_map('trim', explode("\n", $whitelist)));
4912		-
4913		- if (empty($meta_keys)) {
4914		- return array();
4915		- }
4916		-
4917		- $result = array();
4918		-
4919		- foreach ($meta_keys as $key) {
4920		- // Skip empty keys
4921		- if (empty($key)) {
4922		- continue;
4923		- }
4924		-
4925		- $value = get_post_meta($post_id, $key, true);
4926		-
4927		- // Only include non-empty string values
4928		- if (!empty($value) && is_string($value)) {
4929		- $result[$key] = $value;
4930		- } elseif (!empty($value) && is_array($value)) {
4931		- // Handle array values by joining them
4932		- $flat_value = $this->mxchat_flatten_meta_array($value);
4933		- if (!empty($flat_value)) {
4934		- $result[$key] = $flat_value;
4935		- }
4936		- }
4937		- }
4938		-
4939		- return $result;
4940		-}
4941		-
4942		-/**
4943		- * Flatten array meta values into a readable string
4944		- */
4945		-private function mxchat_flatten_meta_array($array, $depth = 0) {
4946		- if ($depth > 3) {
4947		- return ''; // Prevent infinite recursion
4948		- }
4949		-
4950		- $parts = array();
4951		-
4952		- foreach ($array as $key => $value) {
4953		- if (is_string($value) && !empty($value)) {
4954		- $parts[] = $value;
4955		- } elseif (is_array($value)) {
4956		- $nested = $this->mxchat_flatten_meta_array($value, $depth + 1);
4957		- if (!empty($nested)) {
4958		- $parts[] = $nested;
4959		- }
4960		- }
4961		- }
4962		-
4963		- return implode(', ', $parts);
4964		-}
4965		-
4966		-/**
4967		- * Format ACF field values for content extraction
4968		- */
4969		-public function mxchat_format_acf_field_value($value, $field_name = '', $post_id = 0) {
4970		- if (empty($value)) {
4971		- return '';
4972		- }
4973		-
4974		- // Handle WP_Post objects first (THIS IS THE KEY FIX)
4975		- if ($value instanceof WP_Post) {
4976		- return $value->post_title ?: '';
4977		- }
4978		-
4979		- // Handle other WP objects
4980		- if (is_object($value)) {
4981		- if (isset($value->post_title)) {
4982		- return $value->post_title;
4983		- } elseif (isset($value->display_name)) {
4984		- return $value->display_name;
4985		- } elseif (isset($value->name)) {
4986		- return $value->name;
4987		- } elseif (method_exists($value, '__toString')) {
4988		- try {
4989		- return (string) $value;
4990		- } catch (Exception $e) {
4991		- return '';
4992		- }
4993		- }
4994		- // For any other objects, return empty string
4995		- return '';
4996		- }
4997		-
4998		- // Handle different ACF field types
4999		- if (is_array($value)) {
5000		- // Check if it's an image/file field
5001		- if (isset($value['url'])) {
5002		- // Image field - return alt text, title, or caption
5003		- if (!empty($value['alt'])) {
5004		- return $value['alt'];
5005		- } elseif (!empty($value['title'])) {
5006		- return $value['title'];
5007		- } elseif (!empty($value['caption'])) {
5008		- return $value['caption'];
5009		- } else {
5010		- return ''; // Don't include just the URL
5011		- }
5012		- }
5013		-
5014		- // Check if it's a post object or relationship field
5015		- if (isset($value['post_title'])) {
5016		- return $value['post_title'];
5017		- }
5018		-
5019		- // Check if it's a user field
5020		- if (isset($value['display_name'])) {
5021		- return $value['display_name'];
5022		- }
5023		-
5024		- // Check if it's a taxonomy term
5025		- if (isset($value['name']) && isset($value['taxonomy'])) {
5026		- return $value['name'];
5027		- }
5028		-
5029		- // Check if it's a select field with label
5030		- if (isset($value['label'])) {
5031		- return $value['label'];
5032		- }
5033		-
5034		- // Check for repeater field or flexible content
5035		- if (is_numeric(key($value))) {
5036		- $sub_values = array();
5037		- foreach ($value as $sub_item) {
5038		- if (is_array($sub_item)) {
5039		- // For repeater/flexible content, extract text values
5040		- $sub_text = $this->mxchat_extract_text_from_acf_array($sub_item);
5041		- if (!empty($sub_text)) {
5042		- $sub_values[] = $sub_text;
5043		- }
5044		- } elseif ($sub_item instanceof WP_Post) {
5045		- // Handle WP_Post objects in arrays
5046		- $sub_values[] = $sub_item->post_title ?: '';
5047		- } else {
5048		- $sub_values[] = (string) $sub_item;
5049		- }
5050		- }
5051		- return implode(', ', array_filter($sub_values));
5052		- }
5053		-
5054		- // For other arrays, try to extract meaningful text
5055		- $text_values = array();
5056		- foreach ($value as $key => $val) {
5057		- if (is_string($val) && !empty(trim($val))) {
5058		- $text_values[] = trim($val);
5059		- } elseif ($val instanceof WP_Post) {
5060		- // Handle WP_Post objects in associative arrays
5061		- $text_values[] = $val->post_title ?: '';
5062		- } elseif (is_array($val) && isset($val['post_title'])) {
5063		- $text_values[] = $val['post_title'];
5064		- } elseif (is_array($val) && isset($val['name'])) {
5065		- $text_values[] = $val['name'];
5066		- }
5067		- }
5068		-
5069		- return implode(', ', array_filter($text_values));
5070		- }
5071		-
5072		- // Handle boolean values
5073		- if (is_bool($value)) {
5074		- return $value ? 'Yes' : 'No';
5075		- }
5076		-
5077		- // Handle numeric values
5078		- if (is_numeric($value)) {
5079		- return (string) $value;
5080		- }
5081		-
5082		- // Handle string values
5083		- if (is_string($value)) {
5084		- return trim($value);
5085		- }
5086		-
5087		- // For anything else that we can't handle, return empty string
5088		- // This prevents the "Object could not be converted to string" error
5089		- return '';
5090		-}
5091		-
5092		-/**
5093		- * Extract text from complex ACF array structures
5094		- */
5095		-private function mxchat_extract_text_from_acf_array($array) {
5096		- if (!is_array($array)) {
5097		- return '';
5098		- }
5099		-
5100		- $text_parts = array();
5101		-
5102		- foreach ($array as $key => $value) {
5103		- if (is_string($value) && !empty(trim($value))) {
5104		- // Skip keys that are likely to be IDs or technical values
5105		- if (!is_numeric($value) || strlen($value) > 10) {
5106		- $text_parts[] = trim($value);
5107		- }
5108		- } elseif ($value instanceof WP_Post) {
5109		- // Handle WP_Post objects
5110		- $text_parts[] = $value->post_title ?: '';
5111		- } elseif (is_array($value)) {
5112		- if (isset($value['post_title'])) {
5113		- $text_parts[] = $value['post_title'];
5114		- } elseif (isset($value['name'])) {
5115		- $text_parts[] = $value['name'];
5116		- } elseif (isset($value['label'])) {
5117		- $text_parts[] = $value['label'];
5118		- }
5119		- } elseif (is_object($value)) {
5120		- // Handle other objects safely
5121		- if (isset($value->post_title)) {
5122		- $text_parts[] = $value->post_title;
5123		- } elseif (isset($value->name)) {
5124		- $text_parts[] = $value->name;
5125		- } elseif (isset($value->display_name)) {
5126		- $text_parts[] = $value->display_name;
5127		- }
5128		- }
5129		- }
5130		-
5131		- return implode(', ', array_filter($text_parts));
5132		-}
5133		-
5134		-/**
5135		- * Walk an ACF field value tree and collect attachment IDs for any value that
5136		- * resolves to a PDF in the WordPress media library. Handles the three shapes
5137		- * ACF returns for File/Image/URL fields (array with ID+url, integer attachment ID,
5138		- * plain URL string), and recurses through repeater/group/flexible content.
5139		- *
5140		- * @param mixed $value The ACF field value (any depth)
5141		- * @param array $out Accumulator (passed by reference) for attachment IDs
5142		- * @param int $depth Recursion guard
5143		- */
5144		-private function mxchat_collect_pdf_attachment_ids_from_acf_value($value, &$out, $depth = 0) {
5145		- if ($depth > 6) {
5146		- return; // prevent runaway recursion on circular/very-deep structures
5147		- }
5148		-
5149		- if (empty($value)) {
5150		- return;
5151		- }
5152		-
5153		- // Array shapes: ACF File/Image return value=array; repeaters/groups are arrays of arrays
5154		- if (is_array($value)) {
5155		- // Direct File/Image-style array (has 'url' and usually 'ID' + 'mime_type')
5156		- $looks_like_attachment = isset($value['url']) || isset($value['ID']) || isset($value['id']);
5157		- if ($looks_like_attachment) {
5158		- $att_id = 0;
5159		- if (!empty($value['ID']) && is_numeric($value['ID'])) {
5160		- $att_id = (int) $value['ID'];
5161		- } elseif (!empty($value['id']) && is_numeric($value['id'])) {
5162		- $att_id = (int) $value['id'];
5163		- } elseif (!empty($value['url']) && is_string($value['url'])) {
5164		- $att_id = (int) attachment_url_to_postid($value['url']);
5165		- }
5166		-
5167		- $is_pdf = false;
5168		- if (!empty($value['mime_type']) && $value['mime_type'] === 'application/pdf') {
5169		- $is_pdf = true;
5170		- } elseif (!empty($value['subtype']) && strtolower((string) $value['subtype']) === 'pdf') {
5171		- $is_pdf = true;
5172		- } elseif (!empty($value['url']) && is_string($value['url']) && $this->mxchat_url_looks_like_pdf($value['url'])) {
5173		- $is_pdf = true;
5174		- } elseif ($att_id && get_post_mime_type($att_id) === 'application/pdf') {
5175		- $is_pdf = true;
5176		- }
5177		-
5178		- if ($is_pdf && $att_id && get_post_mime_type($att_id) === 'application/pdf') {
5179		- $out[] = $att_id;
5180		- }
5181		- // An array node that represents one attachment doesn't contain other
5182		- // attachments inside it — done with this branch.
5183		- return;
5184		- }
5185		-
5186		- // Recurse: repeater rows, flexible-content layouts, groups, etc.
5187		- foreach ($value as $sub) {
5188		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($sub, $out, $depth + 1);
5189		- }
5190		- return;
5191		- }
5192		-
5193		- // Plain numeric attachment ID (ACF File field set to "Return: ID")
5194		- if (is_numeric($value)) {
5195		- $att_id = (int) $value;
5196		- if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5197		- $out[] = $att_id;
5198		- }
5199		- return;
5200		- }
5201		-
5202		- // Plain string — URL pointing at a PDF (ACF File field set to "Return: URL", or a custom URL/text field)
5203		- if (is_string($value)) {
5204		- $trimmed = trim($value);
5205		- if ($trimmed !== '' && $this->mxchat_url_looks_like_pdf($trimmed)) {
5206		- $att_id = (int) attachment_url_to_postid($trimmed);
5207		- if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5208		- $out[] = $att_id;
5209		- }
5210		- }
5211		- return;
5212		- }
5213		-}
5214		-
5215		-/**
5216		- * Heuristic: does this URL/string look like a PDF reference?
5217		- * Tolerates query strings and fragments (#page=2).
5218		- */
5219		-private function mxchat_url_looks_like_pdf($url) {
5220		- if (!is_string($url) || $url === '') {
5221		- return false;
5222		- }
5223		- // Strip query + fragment before checking extension
5224		- $path = preg_replace('/[?#].*$/', '', $url);
5225		- return (bool) preg_match('/\.pdf$/i', $path);
5226		-}
5227		-
5228		-/**
5229		- * Extract text from a PDF attachment by ID using the bundled Smalot parser.
5230		- * Reads the file directly from disk via get_attached_file (no HTTP fetch).
5231		- * Result is cached on the attachment as post_meta keyed by file mtime so we
5232		- * only parse the same PDF once unless the file changes on disk.
5233		- *
5234		- * @param int $attachment_id
5235		- * @return string Extracted plain text, or '' on failure.
5236		- */
5237		-private function mxchat_extract_pdf_text_by_attachment_id($attachment_id) {
5238		- $attachment_id = (int) $attachment_id;
5239		- if ($attachment_id <= 0) {
5240		- return '';
5241		- }
5242		- if (get_post_mime_type($attachment_id) !== 'application/pdf') {
5243		- return '';
5244		- }
5245		-
5246		- $pdf_path = get_attached_file($attachment_id);
5247		- if (empty($pdf_path) || !file_exists($pdf_path) || !is_readable($pdf_path)) {
5248		- return '';
5249		- }
5250		-
5251		- // Raw-file size cap. Parsing very large PDFs can OOM the request; skip with a log entry
5252		- // and let the rest of the ACF content land in the KB. Filterable for users who need it bigger.
5253		- $default_max_bytes = 25 * 1024 * 1024;
5254		- $max_bytes = (int) apply_filters('mxchat_acf_pdf_max_bytes', $default_max_bytes, $attachment_id, $pdf_path);
5255		- if ($max_bytes > 0) {
5256		- $file_size = @filesize($pdf_path);
5257		- if ($file_size !== false && $file_size > $max_bytes) {
5258		- error_log(sprintf(
5259		- '[mxchat] ACF PDF skipped (over size cap): attachment %d "%s" %d bytes > cap %d',
5260		- $attachment_id,
5261		- basename($pdf_path),
5262		- $file_size,
5263		- $max_bytes
5264		- ));
5265		- return '';
5266		- }
5267		- }
5268		-
5269		- $mtime = @filemtime($pdf_path);
5270		- $cache_meta_key = '_mxchat_acf_pdf_text_v1';
5271		- $cached = get_post_meta($attachment_id, $cache_meta_key, true);
5272		- if (is_array($cached) && isset($cached['mtime'], $cached['text']) && (int) $cached['mtime'] === (int) $mtime) {
5273		- return (string) $cached['text'];
5274		- }
5275		-
5276		- $text = '';
5277		- try {
5278		- if (function_exists('mxchat_load_pdf_parser')) {
5279		- mxchat_load_pdf_parser();
5280		- }
5281		- if (!class_exists('\\Smalot\\PdfParser\\Parser')) {
5282		- return '';
5283		- }
5284		- $parser = new \Smalot\PdfParser\Parser();
5285		- $pdf = $parser->parseFile($pdf_path);
5286		- $pages = $pdf->getPages();
5287		- $page_texts = array();
5288		- foreach ($pages as $page) {
5289		- $page_text = '';
5290		- try {
5291		- $page_text = $page->getText();
5292		- } catch (\Exception $e) {
5293		- $page_text = '';
5294		- }
5295		- if (!empty($page_text)) {
5296		- $page_texts[] = $page_text;
5297		- }
5298		- }
5299		- $text = trim(implode("\n\n", $page_texts));
5300		- } catch (\Exception $e) {
5301		- error_log('[mxchat] ACF PDF extraction failed for attachment ' . $attachment_id . ': ' . $e->getMessage());
5302		- return '';
5303		- } catch (\Throwable $e) {
5304		- error_log('[mxchat] ACF PDF extraction error for attachment ' . $attachment_id . ': ' . $e->getMessage());
5305		- return '';
5306		- }
5307		-
5308		- // Cap per-PDF text to avoid blowing up the embedding payload on enormous PDFs.
5309		- // The chunker downstream will still split this into multiple vectors.
5310		- $max_len = (int) apply_filters('mxchat_acf_pdf_text_max_length', 50000);
5311		- if ($max_len > 0 && strlen($text) > $max_len) {
5312		- $text = substr($text, 0, $max_len);
5313		- }
5314		-
5315		- update_post_meta($attachment_id, $cache_meta_key, array(
5316		- 'mtime' => (int) $mtime,
5317		- 'text' => $text,
5318		- ));
5319		-
5320		- return $text;
5321		-}
5322		-
5323		-/**
5324		- * Handle ACF save - fires after ACF fields are saved
5325		- * This ensures ACF field data is available when syncing to knowledge base
5326		- */
5327		-public function mxchat_handle_acf_save($post_id) {
5328		- // Skip if not a valid post
5329		- if (!$post_id || $post_id === 'options') {
5330		- return;
5331		- }
5332		-
5333		- // Skip autosaves and revisions
	2953	+public function mxchat_handle_post_update($post_id, $post, $update) {
	2954	+ // Basic validation checks
5334	2955	if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5335	2956	return;
5336	2957	}
5337		-
5338		- $post = get_post($post_id);
5339		- if (!$post) {
5340		- return;
5341		- }
5342		-
5343		- $post_type = $post->post_type;
5344		-
5345		- // Check if sync is enabled for this post type
5346		- $should_sync = false;
5347		-
5348		- if ($post_type === 'post' && get_option('mxchat_auto_sync_posts') === '1') {
5349		- $should_sync = true;
5350		- } else if ($post_type === 'page' && get_option('mxchat_auto_sync_pages') === '1') {
5351		- $should_sync = true;
5352		- } else if ($post_type === 'product' && class_exists('WooCommerce')) {
5353		- // WooCommerce products - check if WooCommerce integration is enabled
5354		- $options = get_option('mxchat_options', array());
5355		- if (isset($options['enable_woocommerce_integration']) &&
5356		- ($options['enable_woocommerce_integration'] === '1' || $options['enable_woocommerce_integration'] === 'on')) {
5357		- $should_sync = true;
5358		- }
5359		- } else {
5360		- // Check custom post types
5361		- $option_name = 'mxchat_auto_sync_' . $post_type;
5362		- if (get_option($option_name) === '1') {
5363		- $should_sync = true;
5364		- }
5365		- }
5366		-
5367		- if (!$should_sync) {
5368		- return;
5369		- }
5370		-
5371		- // Only process published posts
	2958	+
	2959	+ // Only process published content
5372	2960	if ($post->post_status !== 'publish') {
5373	2961	return;
5374	2962	}
5375		-
5376		- // Check if this post has any ACF fields - if not, no need to re-sync
5377		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
5378		- if (empty($acf_fields)) {
5379		- return;
5380		- }
5381		-
5382		- // Use a transient to prevent duplicate processing (post_updated may have already run)
5383		- $transient_key = 'mxchat_acf_synced_' . $post_id;
5384		- if (get_transient($transient_key)) {
5385		- return;
5386		- }
5387		- set_transient($transient_key, true, 60); // Prevent re-processing for 60 seconds
5388		-
5389		- // Re-run the sync with ACF data now available
5390		- // We pass $update=true since this is effectively an update with ACF data
5391		- $this->mxchat_handle_post_update($post_id, $post, true);
5392		-}
5393		-
5394		-public function mxchat_handle_post_update($post_id, $post, $update) {
5395		- // Basic validation checks
5396		- if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5397		- return;
5398		- }
5399	2963
5400	2964	$post_type = $post->post_type;
5401	2965
5402	2966	// Check if sync is enabled for this post type
		@@ -5418,276 +2982,80 @@
5418	2982	if (!$should_sync) {
5419	2983	return;
5420	2984	}
5421	2985
5422		- // Check if we have stored the previous status and URL in our transients
5423		- $previous_status_key = 'mxchat_prev_status_' . $post_id;
5424		- $previous_status = get_transient($previous_status_key);
	2986	+ // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
	2987	+ $title = get_the_title($post_id);
	2988	+ $content = get_post_field('post_content', $post_id);
5425	2989
5426		- $previous_url_key = 'mxchat_prev_url_' . $post_id;
5427		- $previous_url = get_transient($previous_url_key);
	2990	+ // Apply WordPress content filters to get properly formatted content
	2991	+ $content = apply_filters('the_content', $content);
5428	2992
5429		- // If the post was previously published but is now not published, remove from knowledge base
5430		- if ($previous_status === 'publish' && $post->post_status !== 'publish') {
5431		- // Use the stored URL from when it was published, or fall back to current permalink
5432		- $source_url = $previous_url ?: get_permalink($post_id);
5433		-
5434		- if ($source_url) {
5435		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
5436		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	2993	+ // Strip tags but preserve structure
	2994	+ $content = wp_strip_all_tags($content);
	2995	+
	2996	+ // Combine title and content
	2997	+ $final_content = $title . "\n\n" . $content;
	2998	+
	2999	+ // For custom post types like job_listing, include additional fields
	3000	+ if ($post_type === 'job_listing') {
	3001	+ // Add job-specific meta if available
	3002	+ $job_location = get_post_meta($post_id, '_job_location', true);
	3003	+ if (!empty($job_location)) {
	3004	+ $final_content .= "\n\nLocation: " . $job_location;
5437	3005	}
5438		-
5439		- // Clean up the transients and exit early
5440		- delete_transient($previous_status_key);
5441		- delete_transient($previous_url_key);
5442		- return;
5443		- }
5444		-
5445		- // Slug/permalink rename while still published: delete the old vectors before upserting new ones.
5446		- // Without this, md5(old_url) vectors (base + chunks) would be orphaned under the stale URL.
5447		- if ($post->post_status === 'publish' && !empty($previous_url)) {
5448		- $current_url = get_permalink($post_id);
5449		- if ($current_url && $current_url !== $previous_url) {
5450		- MxChat_Utils::delete_chunks_for_url($previous_url, 'default');
5451		- }
5452		- }
5453		-
5454		- // Store the current status for next time (if this is an update)
5455		- if ($update) {
5456		- set_transient($previous_status_key, $post->post_status, DAY_IN_SECONDS);
5457		-
5458		- // If the post is currently published, also store its URL
5459		- if ($post->post_status === 'publish') {
5460		- $current_url = get_permalink($post_id);
5461		- set_transient($previous_url_key, $current_url, DAY_IN_SECONDS);
5462		- }
5463		- }
5464		-
5465		- // Only process currently published content for adding/updating
5466		- if ($post->post_status === 'publish') {
5467		- // Get the source URL
5468		- $source_url = get_permalink($post_id);
5469	3006
5470		- // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
5471		- $title = get_the_title($post_id);
5472		- $content = get_post_field('post_content', $post_id);
5473		- $excerpt = get_post_field('post_excerpt', $post_id);
5474		-
5475		- // Remove shortcode tags but preserve content inside them
5476		- $content = $this->strip_shortcode_tags_preserve_content($content);
5477		- $excerpt = $this->strip_shortcode_tags_preserve_content($excerpt);
5478		-
5479		- // Strip tags but preserve structure (don't use 'the_content' filter as it may re-add shortcodes)
5480		- $content = wp_strip_all_tags($content);
5481		-
5482		- // Combine title, short description (if exists), and content
5483		- $final_content = $title . "\n\n";
5484		-
5485		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
5486		- if (!empty($excerpt)) {
5487		- $final_content .= "Short Description: " . wp_strip_all_tags($excerpt) . "\n\n";
5488		- }
5489		-
5490		- $final_content .= $content;
5491		-
5492		- // For WooCommerce products, include pricing and product details
5493		- if ($post_type === 'product' && class_exists('WooCommerce')) {
5494		- $product = wc_get_product($post_id);
5495		-
5496		- if ($product) {
5497		- // Get pricing information
5498		- $regular_price = $product->get_regular_price();
5499		- $sale_price = $product->get_sale_price();
5500		- $price = $product->get_price();
5501		- $sku = $product->get_sku();
5502		-
5503		- // Get currency symbol
5504		- $currency_symbol = get_woocommerce_currency_symbol();
5505		-
5506		- // Add pricing information
5507		- $final_content .= "\n";
5508		- if (!empty($regular_price)) {
5509		- $final_content .= "Price: " . $currency_symbol . $regular_price . "\n";
5510		- } elseif (!empty($price)) {
5511		- $final_content .= "Price: " . $currency_symbol . $price . "\n";
5512		- }
5513		-
5514		- if (!empty($sale_price) && $sale_price !== $regular_price) {
5515		- $final_content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
5516		- }
5517		-
5518		- // Handle variable products - show price range
5519		- if ($product->is_type('variable')) {
5520		- $min_price = $product->get_variation_price('min');
5521		- $max_price = $product->get_variation_price('max');
5522		- if ($min_price !== $max_price) {
5523		- $final_content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
5524		- }
5525		- }
5526		-
5527		- if (!empty($sku)) {
5528		- $final_content .= "SKU: " . $sku . "\n";
5529		- }
5530		-
5531		- // Get product categories
5532		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
5533		- if (!empty($categories) && !is_wp_error($categories)) {
5534		- $final_content .= "Categories: " . implode(', ', $categories) . "\n";
5535		- }
	3007	+ // Get job type terms
	3008	+ $job_types = get_the_terms($post_id, 'job_listing_type');
	3009	+ if (!empty($job_types) && !is_wp_error($job_types)) {
	3010	+ $types = array();
	3011	+ foreach ($job_types as $type) {
	3012	+ $types[] = $type->name;
5536	3013	}
	3014	+ $final_content .= "\n\nJob Type: " . implode(', ', $types);
5537	3015	}
5538		-
5539		- // For custom post types like job_listing, include additional fields
5540		- if ($post_type === 'job_listing') {
5541		- // Add job-specific meta if available
5542		- $job_location = get_post_meta($post_id, '_job_location', true);
5543		- if (!empty($job_location)) {
5544		- $final_content .= "\n\nLocation: " . $job_location;
5545		- }
5546		-
5547		- // Get job type terms
5548		- $job_types = get_the_terms($post_id, 'job_listing_type');
5549		- if (!empty($job_types) && !is_wp_error($job_types)) {
5550		- $types = array();
5551		- foreach ($job_types as $type) {
5552		- $types[] = $type->name;
5553		- }
5554		- $final_content .= "\n\nJob Type: " . implode(', ', $types);
5555		- }
5556		-
5557		- // Get company name if available
5558		- $company_name = get_post_meta($post_id, '_company_name', true);
5559		- if (!empty($company_name)) {
5560		- $final_content .= "\n\nCompany: " . $company_name;
5561		- }
5562		- }
5563		-
5564		- // ADD ACF FIELDS SUPPORT (matches ajax_mxchat_process_selected_content behavior)
5565		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
5566		- if (!empty($acf_fields)) {
5567		- $acf_content_parts = array();
5568		- $pdf_attachment_ids = array();
5569		-
5570		- foreach ($acf_fields as $field_name => $field_value) {
5571		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
5572		- if (!empty($formatted_value)) {
5573		- // Convert field name to readable label
5574		- $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name));
5575		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
5576		- }
5577		-
5578		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
5579		- }
5580		-
5581		- if (!empty($acf_content_parts)) {
5582		- $final_content .= "\n\n" . implode("\n", $acf_content_parts);
5583		- }
5584		-
5585		- // Gate the auto-sync PDF-extraction loop behind an opt-in option.
5586		- // Mirrors the per-batch checkbox the manual content selector has; the
5587		- // 25 MB size cap lives in the shared extractor so it applies in both
5588		- // paths regardless. Default OFF — re-parsing every ACF PDF on every
5589		- // editor save is expensive and most sites don't want it.
5590		- $autosync_extract_acf_pdfs = get_option('mxchat_auto_sync_acf_pdfs', '0') === '1';
5591		- if ($autosync_extract_acf_pdfs && !empty($pdf_attachment_ids)) {
5592		- $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
5593		- $pdf_sections = array();
5594		- foreach ($pdf_attachment_ids as $att_id) {
5595		- $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
5596		- if (!empty($pdf_text)) {
5597		- $pdf_title = get_the_title($att_id);
5598		- $pdf_url = wp_get_attachment_url($att_id);
5599		- $header = 'PDF Attachment';
5600		- if (!empty($pdf_title)) {
5601		- $header .= ': ' . $pdf_title;
5602		- }
5603		- if (!empty($pdf_url)) {
5604		- $header .= ' (' . $pdf_url . ')';
5605		- }
5606		- $pdf_sections[] = $header . "\n" . $pdf_text;
5607		- }
5608		- }
5609		- if (!empty($pdf_sections)) {
5610		- $final_content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
5611		- }
5612		- }
5613		- }
5614		-
5615		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
5616		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
5617		- if (!empty($custom_meta)) {
5618		- $meta_content_parts = array();
5619		-
5620		- foreach ($custom_meta as $meta_key => $meta_value) {
5621		- // Convert meta key to readable label
5622		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
5623		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
5624		- }
5625		-
5626		- if (!empty($meta_content_parts)) {
5627		- $final_content .= "\n\n" . implode("\n", $meta_content_parts);
5628		- }
5629		- }
5630		-
5631		- // Get API key with proper model detection
5632		- $options = get_option('mxchat_options');
5633		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
5634	3016
5635		- if (strpos($selected_model, 'voyage') === 0) {
5636		- $api_key = $options['voyage_api_key'] ?? '';
5637		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
5638		- $api_key = $options['gemini_api_key'] ?? '';
5639		- } else {
5640		- $api_key = $options['api_key'] ?? '';
	3017	+ // Get company name if available
	3018	+ $company_name = get_post_meta($post_id, '_company_name', true);
	3019	+ if (!empty($company_name)) {
	3020	+ $final_content .= "\n\nCompany: " . $company_name;
5641	3021	}
5642		-
5643		- if (empty($api_key)) {
5644		- return;
5645		- }
5646		-
5647		- // Use the centralized utility function for storage
5648		- $result = MxChat_Utils::submit_content_to_db(
5649		- $final_content,
5650		- $source_url,
5651		- $api_key,
5652		- md5($source_url) // Vector ID for Pinecone
5653		- );
5654		-
5655		- // After successful storage, apply role restriction based on tags
5656		- if (!is_wp_error($result)) {
5657		- $this->apply_role_restriction_to_post($post_id, $source_url);
5658		- }
5659	3022	}
5660	3023
5661		- // Clean up the stored previous status if not used above
5662		- if ($previous_status !== 'publish' || $post->post_status === 'publish') {
5663		- delete_transient($previous_status_key);
5664		- delete_transient($previous_url_key);
	3024	+ // Get the source URL
	3025	+ $source_url = get_permalink($post_id);
	3026	+
	3027	+ // Get API key with proper model detection
	3028	+ $options = get_option('mxchat_options');
	3029	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	3030	+
	3031	+ if (strpos($selected_model, 'voyage') === 0) {
	3032	+ $api_key = $options['voyage_api_key'] ?? '';
	3033	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	3034	+ $api_key = $options['gemini_api_key'] ?? '';
	3035	+ } else {
	3036	+ $api_key = $options['api_key'] ?? '';
5665	3037	}
5666		-}
5667		-
5668		-/**
5669		- * Store the post status and URL before update to detect status transitions
5670		- * This runs before the post is actually updated in the database
5671		- */
5672		-public function mxchat_store_pre_update_status($post_id, $data) {
5673		- // Get the current post from database (before update)
5674		- $current_post = get_post($post_id);
5675	3038
5676		- if ($current_post) {
5677		- // Store the current status temporarily
5678		- $status_key = 'mxchat_prev_status_' . $post_id;
5679		- set_transient($status_key, $current_post->post_status, HOUR_IN_SECONDS);
5680		-
5681		- // If the post is currently published, also store its URL
5682		- if ($current_post->post_status === 'publish') {
5683		- $url_key = 'mxchat_prev_url_' . $post_id;
5684		- $current_url = get_permalink($post_id);
5685		- set_transient($url_key, $current_url, HOUR_IN_SECONDS);
5686		- }
	3039	+ if (empty($api_key)) {
	3040	+ error_log('MxChat Auto-sync: No API key configured for embedding model');
	3041	+ return;
5687	3042	}
	3043	+
	3044	+ // Use the centralized utility function for storage
	3045	+ $result = MxChat_Utils::submit_content_to_db(
	3046	+ $final_content,
	3047	+ $source_url,
	3048	+ $api_key,
	3049	+ md5($source_url) // Vector ID for Pinecone
	3050	+ );
	3051	+
	3052	+ if (is_wp_error($result)) {
	3053	+ error_log('MxChat Auto-sync failed for post ' . $post_id . ': ' . $result->get_error_message());
	3054	+ }
5688	3055	}
5689	3056
	3057	+
5690	3058	public function mxchat_handle_post_delete($post_id) {
5691	3059	// Get post data before it's deleted
5692	3060	$post = get_post($post_id);
5693	3061
		@@ -5717,51 +3085,85 @@
5717	3085	if (!$should_sync) {
5718	3086	return;
5719	3087	}
5720	3088
5721		- // Resolve the pre-trash URL. wp_trash_post renames the slug with "__trashed" before firing
5722		- // this hook, so get_permalink() here would return the trashed URL and md5() would miss the
5723		- // real vector IDs stored under the original URL.
5724		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
	3089	+ // Get the URL before post is deleted
	3090	+ $source_url = get_permalink($post_id);
5725	3091	if (!$source_url) {
5726		- //error_log('MXChat: Failed to resolve source URL for post ' . $post_id);
	3092	+ error_log('MXChat: Failed to get permalink for post ' . $post_id);
5727	3093	return;
5728	3094	}
5729	3095
5730		- // Use chunk-aware deletion (handles both chunked and non-chunked content)
5731		- $delete_result = MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	3096	+ // Check if Pinecone is enabled
	3097	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	3098	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5732	3099
5733		- if (is_wp_error($delete_result)) {
5734		- //error_log('MXChat: Chunk-aware deletion failed for URL: ' . $source_url . ' - ' . $delete_result->get_error_message());
	3100	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3101	+ // Delete from Pinecone
	3102	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
	3103	+ } else {
	3104	+ // Delete from WordPress DB
	3105	+ global $wpdb;
	3106	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	3107	+
	3108	+ $result = $wpdb->delete(
	3109	+ $table_name,
	3110	+ array('source_url' => $source_url),
	3111	+ array('%s')
	3112	+ );
	3113	+
	3114	+ if ($result === false) {
	3115	+ error_log('MXChat: WordPress DB deletion failed for URL: ' . $source_url);
	3116	+ }
5735	3117	}
	3118	+}
5736	3119
5737		- delete_transient('mxchat_prev_url_' . $post_id);
5738		- delete_transient('mxchat_prev_status_' . $post_id);
5739		-}
5740	3120
5741		-/**
5742		- * Resolve the source URL for a post being trashed/deleted.
5743		- *
5744		- * Why: wp_trash_post appends "__trashed" to the slug before the wp_trash_post action fires, so
5745		- * get_permalink() returns a URL whose md5() won't match the vector IDs stored in Pinecone or
5746		- * the source_url rows in the WP DB. Prefer the URL captured by mxchat_store_pre_update_status
5747		- * (runs on pre_post_update, before the rename); fall back to stripping the __trashed suffix.
5748		- */
5749		-private function mxchat_resolve_pre_trash_url($post_id) {
5750		- $previous_url = get_transient('mxchat_prev_url_' . $post_id);
5751		- if (!empty($previous_url)) {
5752		- return $previous_url;
5753		- }
	3121	+ /**
	3122	+ * Deletes data from Pinecone using a source URL
	3123	+ */
	3124	+ public function mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options) {
	3125	+ $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
	3126	+ $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
5754	3127
5755		- $current = get_permalink($post_id);
5756		- if (!$current) {
5757		- return '';
5758		- }
5759		- return preg_replace('#__trashed(/?)$#', '$1', $current);
5760		-}
	3128	+ if (empty($host) || empty($api_key)) {
	3129	+ //error_log('MXChat: Pinecone deletion failed - missing configuration');
	3130	+ return false;
	3131	+ }
5761	3132
	3133	+ $api_endpoint = "https://{$host}/vectors/delete";
	3134	+ $vector_id = md5($source_url);
5762	3135
	3136	+ $request_body = array(
	3137	+ 'ids' => array($vector_id)
	3138	+ );
5763	3139
	3140	+ $response = wp_remote_post($api_endpoint, array(
	3141	+ 'headers' => array(
	3142	+ 'Api-Key' => $api_key,
	3143	+ 'accept' => 'application/json',
	3144	+ 'content-type' => 'application/json'
	3145	+ ),
	3146	+ 'body' => wp_json_encode($request_body),
	3147	+ 'timeout' => 30
	3148	+ ));
	3149	+
	3150	+ if (is_wp_error($response)) {
	3151	+ //error_log('MXChat: Pinecone deletion error - ' . $response->get_error_message());
	3152	+ return false;
	3153	+ }
	3154	+
	3155	+ $response_code = wp_remote_retrieve_response_code($response);
	3156	+ if ($response_code !== 200) {
	3157	+ //error_log('MXChat: Pinecone deletion failed with status ' . $response_code);
	3158	+ return false;
	3159	+ }
	3160	+
	3161	+ return true;
	3162	+ }
	3163	+
	3164	+
	3165	+
5764	3166	public function mxchat_handle_product_change($post_id, $post, $update) {
5765	3167	if ($post->post_type !== 'product') {
5766	3168	return;
5767	3169	}
		@@ -5783,12 +3185,11 @@
5783	3185	if (!isset($this->options['enable_woocommerce_integration']) ||
5784	3186	!in_array($this->options['enable_woocommerce_integration'], ['1', 'on'])) {
5785	3187	return;
5786	3188	}
5787		-
	3189	+
5788	3190	$source_url = get_permalink($product->get_id());
5789		- $product_id = $product->get_id();
5790		-
	3191	+
5791	3192	// Build product content
5792	3193	$title = $product->get_name();
5793	3194	$description = $product->get_description();
5794	3195	$short_description = $product->get_short_description();
		@@ -5793,92 +3194,35 @@
5793	3194	$description = $product->get_description();
5794	3195	$short_description = $product->get_short_description();
5795	3196	$regular_price = $product->get_regular_price();
5796	3197	$sale_price = $product->get_sale_price();
5797		- $price = $product->get_price();
5798	3198	$sku = $product->get_sku();
5799		-
5800		- // Get currency symbol
5801		- $currency_symbol = get_woocommerce_currency_symbol();
5802		-
	3199	+
5803	3200	// Format content consistently
5804	3201	$content = $title . "\n\n";
5805		-
5806		- if (!empty($short_description)) {
5807		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
5808		- }
5809		-
	3202	+
5810	3203	if (!empty($description)) {
5811	3204	$content .= wp_strip_all_tags($description) . "\n\n";
5812	3205	}
5813		-
5814		- // Add pricing information
5815		- if (!empty($regular_price)) {
5816		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
5817		- } elseif (!empty($price)) {
5818		- $content .= "Price: " . $currency_symbol . $price . "\n";
	3206	+
	3207	+ if (!empty($short_description)) {
	3208	+ $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
5819	3209	}
5820		-
5821		- if (!empty($sale_price) && $sale_price !== $regular_price) {
5822		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
	3210	+
	3211	+ $content .= "Price: $" . $regular_price . "\n";
	3212	+
	3213	+ if (!empty($sale_price)) {
	3214	+ $content .= "Sale Price: $" . $sale_price . "\n";
5823	3215	}
5824		-
5825		- // Handle variable products - show price range
5826		- if ($product->is_type('variable')) {
5827		- $min_price = $product->get_variation_price('min');
5828		- $max_price = $product->get_variation_price('max');
5829		- if ($min_price !== $max_price) {
5830		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
5831		- }
5832		- }
5833		-
	3216	+
5834	3217	if (!empty($sku)) {
5835	3218	$content .= "SKU: " . $sku . "\n";
5836	3219	}
5837		-
5838		- // Get product categories
5839		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
5840		- if (!empty($categories) && !is_wp_error($categories)) {
5841		- $content .= "Categories: " . implode(', ', $categories) . "\n";
5842		- }
5843		-
5844		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
5845		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
5846		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
5847		- foreach ($custom_tabs as $tab) {
5848		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
5849		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
5850		-
5851		- if (!empty($tab_title) && !empty($tab_content)) {
5852		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
5853		- }
5854		- }
5855		- }
5856		-
5857		- // Also check for reusable/saved tabs applied to this product
5858		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
5859		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
5860		- // Get the saved tabs option
5861		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
5862		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
5863		- foreach ($applied_saved_tabs as $saved_tab_id) {
5864		- if (isset($saved_tabs[$saved_tab_id])) {
5865		- $tab = $saved_tabs[$saved_tab_id];
5866		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
5867		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
5868		-
5869		- if (!empty($tab_title) && !empty($tab_content)) {
5870		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
5871		- }
5872		- }
5873		- }
5874		- }
5875		- }
5876		-
	3220	+
5877	3221	// Get API key with proper model detection
5878	3222	$options = get_option('mxchat_options');
5879	3223	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
5880		-
	3224	+
5881	3225	if (strpos($selected_model, 'voyage') === 0) {
5882	3226	$api_key = $options['voyage_api_key'] ?? '';
5883	3227	} elseif (strpos($selected_model, 'gemini-embedding') === 0) {
5884	3228	$api_key = $options['gemini_api_key'] ?? '';
		@@ -5884,14 +3228,14 @@
5884	3228	$api_key = $options['gemini_api_key'] ?? '';
5885	3229	} else {
5886	3230	$api_key = $options['api_key'] ?? '';
5887	3231	}
5888		-
	3232	+
5889	3233	if (empty($api_key)) {
5890		- //error_log('MxChat Auto-sync: No API key configured for embedding model');
	3234	+ error_log('MxChat Auto-sync: No API key configured for embedding model');
5891	3235	return;
5892	3236	}
5893		-
	3237	+
5894	3238	// Use the centralized utility function for storage
5895	3239	$result = MxChat_Utils::submit_content_to_db(
5896	3240	$content,
5897	3241	$source_url,
		@@ -5897,16 +3241,11 @@
5897	3241	$source_url,
5898	3242	$api_key,
5899	3243	md5($source_url) // Vector ID for Pinecone
5900	3244	);
5901		-
5902		- // After successful storage, apply role restriction based on tags
5903		- if (!is_wp_error($result)) {
5904		- $this->apply_role_restriction_to_post($product_id, $source_url);
5905		- }
5906		-
	3245	+
5907	3246	if (is_wp_error($result)) {
5908		- //error_log('MxChat WooCommerce sync failed for product ' . $product_id . ': ' . $result->get_error_message());
	3247	+ error_log('MxChat WooCommerce sync failed for product ' . $product->get_id() . ': ' . $result->get_error_message());
5909	3248	}
5910	3249	}
5911	3250
5912	3251	public function mxchat_handle_product_delete($post_id) {
		@@ -5913,1112 +3252,30 @@
5913	3252	if (get_post_type($post_id) !== 'product') {
5914	3253	return;
5915	3254	}
5916	3255
5917		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
5918		- if (!$source_url) {
5919		- return;
5920		- }
	3256	+ $source_url = get_permalink($post_id);
5921	3257
5922		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
5923		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
5924		-
5925		- delete_transient('mxchat_prev_url_' . $post_id);
5926		- delete_transient('mxchat_prev_status_' . $post_id);
5927		-}
5928		-
5929		-/**
5930		- * Handle individual Pinecone content deletion
5931		- */
5932		-public function mxchat_handle_pinecone_prompt_delete() {
5933		- // Check permissions
5934		- if (!current_user_can('manage_options')) {
5935		- wp_die(esc_html__('You do not have sufficient permissions.', 'mxchat'));
5936		- }
5937		-
5938		- // Verify nonce
5939		- if (!isset($_GET['_wpnonce']) || !wp_verify_nonce($_GET['_wpnonce'], 'mxchat_delete_pinecone_prompt_nonce')) {
5940		- wp_die(esc_html__('Security check failed.', 'mxchat'));
5941		- }
5942		-
5943		- $vector_id = isset($_GET['vector_id']) ? sanitize_text_field($_GET['vector_id']) : '';
5944		-
5945		- if (empty($vector_id)) {
5946		- set_transient('mxchat_admin_notice_error',
5947		- esc_html__('Invalid vector ID.', 'mxchat'),
5948		- 30
5949		- );
5950		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
5951		- exit;
5952		- }
5953		-
5954		- // Get Pinecone settings
	3258	+ // Check if Pinecone is enabled
5955	3259	$pinecone_options = get_option('mxchat_pinecone_addon_options', array());
5956	3260	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5957		-
5958		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
5959		- set_transient('mxchat_admin_notice_error',
5960		- esc_html__('Pinecone is not properly configured.', 'mxchat'),
5961		- 30
5962		- );
5963		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
5964		- exit;
5965		- }
5966		-
5967		- // Delete from Pinecone
5968		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
5969		- $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
5970		- $vector_id,
5971		- $pinecone_options['mxchat_pinecone_api_key'],
5972		- $pinecone_options['mxchat_pinecone_host']
5973		- );
5974		-
5975		- if ($result['success']) {
5976		- // No cache clearing needed since we removed caching
5977		- set_transient('mxchat_admin_notice_success',
5978		- esc_html__('Entry deleted successfully from Pinecone.', 'mxchat'),
5979		- 30
5980		- );
5981		- } else {
5982		- set_transient('mxchat_admin_notice_error',
5983		- esc_html__('Failed to delete entry: ', 'mxchat') . $result['message'],
5984		- 30
5985		- );
5986		- }
5987		-
5988		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
5989		- exit;
5990		-}
5991		-/**
5992		- * Handle individual Pinecone content deletion via AJAX
5993		- */
5994		-public function ajax_mxchat_delete_pinecone_prompt() {
5995		- // Verify nonce and permissions
5996		- if (!check_ajax_referer('mxchat_delete_pinecone_prompt_nonce', 'nonce', false)) {
5997		- wp_send_json_error('Invalid nonce');
5998		- exit;
5999		- }
6000		-
6001		- if (!current_user_can('manage_options')) {
6002		- wp_send_json_error('Unauthorized access');
6003		- exit;
6004		- }
6005		-
6006		- $vector_id = isset($_POST['vector_id']) ? sanitize_text_field($_POST['vector_id']) : '';
6007		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6008		-
6009		- if (empty($vector_id)) {
6010		- wp_send_json_error('Missing vector ID');
6011		- exit;
6012		- }
6013		-
6014		- // Get bot-specific Pinecone settings
6015		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6016		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6017		-
6018		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6019		-
6020		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
6021		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
6022		- exit;
6023		- }
6024		-
6025		- // Delete from the correct Pinecone index
6026		- $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
6027		- $vector_id,
6028		- $pinecone_options['mxchat_pinecone_api_key'],
6029		- $pinecone_options['mxchat_pinecone_host']
6030		- );
6031		-
6032		- if ($result['success']) {
6033		- // No cache clearing needed since we removed caching
6034		- wp_send_json_success(array(
6035		- 'message' => 'Entry deleted successfully from Pinecone',
6036		- 'vector_id' => $vector_id,
6037		- 'bot_id' => $bot_id
6038		- ));
6039		- } else {
6040		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete from Pinecone: ' . $result['message'], array('vector_id' => $vector_id, 'bot_id' => $bot_id));
6041		- wp_send_json_error('Failed to delete from Pinecone: ' . $result['message']);
6042		- }
6043		-
6044		- exit;
6045		-}
6046	3261
6047		-/**
6048		- * Handle deletion of all chunks for a given source URL via AJAX
6049		- * Follows the same pattern as ajax_mxchat_delete_pinecone_prompt
6050		- */
6051		-public function ajax_mxchat_delete_chunks_by_url() {
6052		- // Verify nonce and permissions
6053		- if (!check_ajax_referer('mxchat_delete_chunks_nonce', 'nonce', false)) {
6054		- wp_send_json_error('Invalid nonce');
6055		- exit;
6056		- }
6057		-
6058		- if (!current_user_can('manage_options')) {
6059		- wp_send_json_error('Unauthorized access');
6060		- exit;
6061		- }
6062		-
6063		- $source_url = isset($_POST['source_url']) ? esc_url_raw($_POST['source_url']) : '';
6064		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
6065		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6066		-
6067		- if (empty($source_url)) {
6068		- wp_send_json_error('Missing source URL');
6069		- exit;
6070		- }
6071		-
6072		- // Generate the base vector ID from the source URL (same as how chunks are created)
6073		- $base_vector_id = md5($source_url);
6074		-
6075		- if ($data_source === 'pinecone') {
6076		- // Get bot-specific Pinecone settings (same as working delete function)
6077		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6078		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6079		-
6080		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6081		-
6082		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
6083		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
6084		- exit;
6085		- }
6086		-
6087		- $api_key = $pinecone_options['mxchat_pinecone_api_key'];
6088		- $host = $pinecone_options['mxchat_pinecone_host'];
6089		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
6090		-
6091		- // Collect all vector IDs to delete
6092		- $vectors_to_delete = array();
6093		-
6094		- // Add the original single-vector ID (for non-chunked content)
6095		- $vectors_to_delete[] = $base_vector_id;
6096		-
6097		- // Use Pinecone list API to find all chunk vectors with this prefix
6098		- // NOTE: Pinecone List API is a GET request with query parameters, not POST
6099		- $prefix = $base_vector_id . '_chunk_';
6100		-
6101		- $query_params = array(
6102		- 'prefix' => $prefix,
6103		- 'limit' => 100
6104		- );
6105		-
6106		- if (!empty($namespace)) {
6107		- $query_params['namespace'] = $namespace;
6108		- }
6109		-
6110		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
6111		-
6112		- $list_response = wp_remote_get($list_url, array(
6113		- 'headers' => array(
6114		- 'Api-Key' => $api_key,
6115		- 'accept' => 'application/json'
6116		- ),
6117		- 'timeout' => 30
6118		- ));
6119		-
6120		- if (!is_wp_error($list_response)) {
6121		- $list_body_response = wp_remote_retrieve_body($list_response);
6122		- $list_data = json_decode($list_body_response, true);
6123		- if (!empty($list_data['vectors'])) {
6124		- foreach ($list_data['vectors'] as $vector) {
6125		- if (isset($vector['id'])) {
6126		- $vectors_to_delete[] = $vector['id'];
6127		- }
6128		- }
6129		- }
6130		- }
6131		-
6132		- if (empty($vectors_to_delete)) {
6133		- wp_send_json_success(array(
6134		- 'message' => 'No vectors found to delete',
6135		- 'source_url' => $source_url
6136		- ));
6137		- exit;
6138		- }
6139		-
6140		- // Delete all vectors using the same endpoint as the working function
6141		- $delete_url = "https://{$host}/vectors/delete";
6142		-
6143		- $delete_body = array(
6144		- 'ids' => $vectors_to_delete
6145		- );
6146		-
6147		- if (!empty($namespace)) {
6148		- $delete_body['namespace'] = $namespace;
6149		- }
6150		-
6151		- $delete_response = wp_remote_post($delete_url, array(
6152		- 'headers' => array(
6153		- 'Api-Key' => $api_key,
6154		- 'accept' => 'application/json',
6155		- 'content-type' => 'application/json'
6156		- ),
6157		- 'body' => wp_json_encode($delete_body),
6158		- 'timeout' => 30
6159		- ));
6160		-
6161		- if (is_wp_error($delete_response)) {
6162		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete chunks from Pinecone: ' . $delete_response->get_error_message(), array('source_url' => $source_url));
6163		- wp_send_json_error('Failed to delete from Pinecone: ' . $delete_response->get_error_message());
6164		- exit;
6165		- }
6166		-
6167		- $response_code = wp_remote_retrieve_response_code($delete_response);
6168		-
6169		- if ($response_code !== 200) {
6170		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Pinecone API error (HTTP ' . $response_code . ')', array('source_url' => $source_url));
6171		- wp_send_json_error('Pinecone API error (HTTP ' . $response_code . ')');
6172		- exit;
6173		- }
6174		-
6175		- wp_send_json_success(array(
6176		- 'message' => 'All chunks deleted successfully from Pinecone',
6177		- 'source_url' => $source_url,
6178		- 'deleted_count' => count($vectors_to_delete)
6179		- ));
6180		-
	3262	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3263	+ // Delete from Pinecone
	3264	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
6181	3265	} else {
6182		- // WordPress database deletion
	3266	+ // Delete from WordPress DB
6183	3267	global $wpdb;
6184	3268	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6185		-
6186		- $result = $wpdb->delete(
6187		- $table_name,
6188		- array('source_url' => $source_url),
6189		- array('%s')
6190		- );
6191		-
6192		- if ($result === false) {
6193		- MxChat_Admin::mxchat_log_debug('database_error', 'Failed to delete from database: ' . $wpdb->last_error, array('source_url' => $source_url));
6194		- wp_send_json_error('Failed to delete from database: ' . $wpdb->last_error);
6195		- exit;
6196		- }
6197		-
6198		- wp_send_json_success(array(
6199		- 'message' => 'All chunks deleted successfully from database',
6200		- 'source_url' => $source_url,
6201		- 'deleted_count' => $result
6202		- ));
6203		- }
6204		-
6205		- exit;
6206		-}
6207		-
6208		-/**
6209		- * Handle individual WordPress database content deletion via AJAX
6210		- * Mirrors the Pinecone delete handler but for WordPress database entries
6211		- */
6212		-public function ajax_mxchat_delete_wordpress_prompt() {
6213		- // Verify nonce and permissions
6214		- if (!check_ajax_referer('mxchat_delete_wordpress_prompt_nonce', 'nonce', false)) {
6215		- wp_send_json_error('Invalid nonce');
6216		- exit;
6217		- }
6218		-
6219		- if (!current_user_can('manage_options')) {
6220		- wp_send_json_error('Unauthorized access');
6221		- exit;
6222		- }
6223		-
6224		- $entry_id = isset($_POST['entry_id']) ? intval($_POST['entry_id']) : 0;
6225		-
6226		- if (empty($entry_id)) {
6227		- wp_send_json_error('Missing entry ID');
6228		- exit;
6229		- }
6230		-
6231		- global $wpdb;
6232		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6233		-
6234		- // Clear cache for this entry
6235		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
6236		-
6237		- // Delete from database
6238		- $result = $wpdb->delete(
6239		- $table_name,
6240		- array('id' => $entry_id),
6241		- array('%d')
6242		- );
6243		-
6244		- if ($result !== false) {
6245		- wp_send_json_success(array(
6246		- 'message' => 'Entry deleted successfully',
6247		- 'entry_id' => $entry_id
6248		- ));
6249		- } else {
6250		- wp_send_json_error('Failed to delete entry from database');
6251		- }
6252		-
6253		- exit;
6254		-}
6255		-
6256		-/**
6257		- * Handle bulk deletion of knowledge entries via AJAX
6258		- * Supports both Pinecone and WordPress database entries
6259		- */
6260		-public function ajax_mxchat_bulk_delete_knowledge() {
6261		- // Verify nonce and permissions
6262		- if (!check_ajax_referer('mxchat_bulk_delete_knowledge_nonce', 'nonce', false)) {
6263		- wp_send_json_error('Invalid nonce');
6264		- exit;
6265		- }
6266		-
6267		- if (!current_user_can('manage_options')) {
6268		- wp_send_json_error('Unauthorized access');
6269		- exit;
6270		- }
6271		-
6272		- $entries = isset($_POST['entries']) ? $_POST['entries'] : array();
6273		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6274		-
6275		- if (empty($entries) || !is_array($entries)) {
6276		- wp_send_json_error('No entries provided');
6277		- exit;
6278		- }
6279		-
6280		- // Extend execution time — bulk Pinecone operations can take a while
6281		- if (function_exists('set_time_limit')) {
6282		- set_time_limit(120);
6283		- }
6284		-
6285		- $success_ids = array();
6286		- $failed_ids = array();
6287		- $errors = array();
6288		-
6289		- global $wpdb;
6290		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6291		-
6292		- // Get Pinecone manager for Pinecone deletions
6293		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6294		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6295		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6296		-
6297		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
6298		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
6299		-
6300		- // =============================================
6301		- // PHASE 1: Collect all Pinecone vector IDs
6302		- // and separate WordPress entries
6303		- // =============================================
6304		- $pinecone_entry_ids = array(); // entry IDs that are pinecone-sourced
6305		- $wordpress_entries = array(); // entries for WordPress DB deletion
6306		- $all_vector_ids = array(); // all pinecone vector IDs to delete in one batch
6307		-
6308		- foreach ($entries as $entry) {
6309		- $entry_id = sanitize_text_field($entry['id'] ?? '');
6310		- $source = sanitize_text_field($entry['source'] ?? 'wordpress');
6311		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
6312		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
6313		-
6314		- if (empty($entry_id)) {
6315		- continue;
6316		- }
6317		-
6318		- if ($source === 'pinecone') {
6319		- if (!$use_pinecone || empty($api_key)) {
6320		- $failed_ids[] = $entry_id;
6321		- $errors[] = "Pinecone not configured for entry: $entry_id";
6322		- continue;
6323		- }
6324		-
6325		- $pinecone_entry_ids[] = $entry_id;
6326		-
6327		- if ($is_group && !empty($source_url)) {
6328		- // Grouped/chunked entry: collect base ID + chunk IDs via List API
6329		- $base_vector_id = md5($source_url);
6330		- $all_vector_ids[] = $base_vector_id;
6331		-
6332		- $list_url = 'https://' . $host . '/vectors/list?prefix=' . urlencode($base_vector_id . '_chunk_') . '&limit=100';
6333		- $list_response = wp_remote_get($list_url, array(
6334		- 'headers' => array(
6335		- 'Api-Key' => $api_key,
6336		- 'accept' => 'application/json'
6337		- ),
6338		- 'timeout' => 30
6339		- ));
6340		-
6341		- if (!is_wp_error($list_response)) {
6342		- $list_body = json_decode(wp_remote_retrieve_body($list_response), true);
6343		- if (!empty($list_body['vectors']) && is_array($list_body['vectors'])) {
6344		- foreach ($list_body['vectors'] as $vector) {
6345		- if (isset($vector['id'])) {
6346		- $all_vector_ids[] = $vector['id'];
6347		- }
6348		- }
6349		- }
6350		- }
6351		- } else {
6352		- // Single entry: the entry_id IS the vector ID
6353		- $all_vector_ids[] = $entry_id;
6354		- }
6355		- } else {
6356		- $wordpress_entries[] = $entry;
6357		- }
6358		- }
6359		-
6360		- // =============================================
6361		- // PHASE 2: Single batch delete to Pinecone
6362		- // =============================================
6363		- if (!empty($all_vector_ids)) {
6364		- $all_vector_ids = array_values(array_unique($all_vector_ids));
6365		- $pinecone_success = true;
6366		- $batches = array_chunk($all_vector_ids, 100);
6367		-
6368		- foreach ($batches as $batch) {
6369		- $delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
6370		- 'headers' => array(
6371		- 'Api-Key' => $api_key,
6372		- 'accept' => 'application/json',
6373		- 'content-type' => 'application/json'
6374		- ),
6375		- 'body' => wp_json_encode(array('ids' => $batch)),
6376		- 'timeout' => 60
6377		- ));
6378		-
6379		- if (is_wp_error($delete_response)) {
6380		- $pinecone_success = false;
6381		- $errors[] = 'Pinecone batch deletion failed: ' . $delete_response->get_error_message();
6382		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed: ' . $delete_response->get_error_message());
6383		- } else {
6384		- $response_code = wp_remote_retrieve_response_code($delete_response);
6385		- if ($response_code !== 200) {
6386		- $pinecone_success = false;
6387		- $response_body = wp_remote_retrieve_body($delete_response);
6388		- $errors[] = "Pinecone API error (HTTP $response_code)";
6389		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed (HTTP ' . $response_code . ')', array('response' => substr($response_body, 0, 200)));
6390		- }
6391		- }
6392		- }
6393		-
6394		- // Mark all pinecone entries based on batch result
6395		- foreach ($pinecone_entry_ids as $eid) {
6396		- if ($pinecone_success) {
6397		- $success_ids[] = $eid;
6398		- } else {
6399		- $failed_ids[] = $eid;
6400		- }
6401		- }
6402		- }
6403		-
6404		- // =============================================
6405		- // PHASE 3: WordPress database deletions
6406		- // =============================================
6407		- foreach ($wordpress_entries as $entry) {
6408		- $entry_id = sanitize_text_field($entry['id'] ?? '');
6409		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
6410		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
6411		-
6412		- if (empty($entry_id)) {
6413		- continue;
6414		- }
6415		-
6416		- try {
6417		- if ($is_group && !empty($source_url)) {
6418		- $result = $wpdb->delete(
6419		- $table_name,
6420		- array('source_url' => $source_url),
6421		- array('%s')
6422		- );
6423		- } else {
6424		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
6425		- $result = $wpdb->delete(
6426		- $table_name,
6427		- array('id' => intval($entry_id)),
6428		- array('%d')
6429		- );
6430		- }
6431		-
6432		- if ($result !== false) {
6433		- $success_ids[] = $entry_id;
6434		- } else {
6435		- $failed_ids[] = $entry_id;
6436		- $errors[] = "Database error for entry: $entry_id";
6437		- }
6438		- } catch (Exception $e) {
6439		- $failed_ids[] = $entry_id;
6440		- $errors[] = $e->getMessage();
6441		- }
6442		- }
6443		-
6444		- wp_send_json_success(array(
6445		- 'success_ids' => $success_ids,
6446		- 'failed_ids' => $failed_ids,
6447		- 'errors' => $errors,
6448		- 'total_processed' => count($success_ids) + count($failed_ids)
6449		- ));
6450		-
6451		- exit;
6452		-}
6453		-
6454		-/**
6455		- * Get hierarchical roles for dropdown
6456		- */
6457		-public function mxchat_get_role_options() {
6458		- return array(
6459		- 'public' => __('Public (Everyone)', 'mxchat'),
6460		- 'logged_in' => __('Logged In Users', 'mxchat'),
6461		- 'subscriber' => __('Subscribers & Above', 'mxchat'),
6462		- 'contributor' => __('Contributors & Above', 'mxchat'),
6463		- 'author' => __('Authors & Above', 'mxchat'),
6464		- 'editor' => __('Editors & Above', 'mxchat'),
6465		- 'administrator' => __('Administrators Only', 'mxchat')
6466		- );
6467		-}
6468		-
6469		-/**
6470		- * Check if user has access to content based on role restriction
6471		- */
6472		-public function mxchat_user_has_content_access($role_restriction) {
6473		- // Public content is always accessible
6474		- if ($role_restriction === 'public' || empty($role_restriction)) {
6475		- return true;
6476		- }
6477		-
6478		- // Check if user is logged in for logged_in restriction
6479		- if ($role_restriction === 'logged_in') {
6480		- return is_user_logged_in();
6481		- }
6482		-
6483		- // If not logged in, no access to role-restricted content
6484		- if (!is_user_logged_in()) {
6485		- return false;
6486		- }
6487		-
6488		- $user = wp_get_current_user();
6489		- $user_roles = $user->roles;
6490		-
6491		- if (empty($user_roles)) {
6492		- return false;
6493		- }
6494		-
6495		- // Define role hierarchy (higher number = higher access)
6496		- $hierarchy = array(
6497		- 'subscriber' => 1,
6498		- 'contributor' => 2,
6499		- 'author' => 3,
6500		- 'editor' => 4,
6501		- 'administrator' => 5
6502		- );
6503		-
6504		- // Get required level
6505		- $required_level = isset($hierarchy[$role_restriction]) ? $hierarchy[$role_restriction] : 0;
6506		-
6507		- // Check if user has required level or higher
6508		- foreach ($user_roles as $user_role) {
6509		- $user_level = isset($hierarchy[$user_role]) ? $hierarchy[$user_role] : 0;
6510		- if ($user_level >= $required_level) {
6511		- return true;
6512		- }
6513		- }
6514		-
6515		- return false;
6516		-}
6517		-
6518		-/**
6519		- * Handle role restriction updates via AJAX
6520		- * Removed cache clearing call since we removed caching
6521		- */
6522		-public function ajax_mxchat_update_role_restriction() {
6523		- // Verify nonce and permissions
6524		- if (!check_ajax_referer('mxchat_update_role_nonce', 'nonce', false)) {
6525		- wp_send_json_error('Invalid nonce');
6526		- exit;
6527		- }
6528		-
6529		- if (!current_user_can('manage_options')) {
6530		- wp_send_json_error('Unauthorized access');
6531		- exit;
6532		- }
6533		-
6534		- $entry_id = isset($_POST['entry_id']) ? sanitize_text_field($_POST['entry_id']) : '';
6535		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
6536		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
6537		-
6538		- if (empty($entry_id)) {
6539		- wp_send_json_error('Invalid entry ID');
6540		- exit;
6541		- }
6542		-
6543		- // Get knowledge manager instance to validate role restriction
6544		- $knowledge_manager = MxChat_Knowledge_Manager::get_instance();
6545		- $valid_roles = array_keys($knowledge_manager->mxchat_get_role_options());
6546		- if (!in_array($role_restriction, $valid_roles)) {
6547		- wp_send_json_error('Invalid role restriction');
6548		- exit;
6549		- }
6550		-
6551		- global $wpdb;
6552		-
6553		- if ($data_source === 'pinecone') {
6554		- // Handle Pinecone role restriction (stored separately in WordPress table)
6555		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6556	3269
6557		- // Use REPLACE to insert or update the role restriction
6558		- $result = $wpdb->replace(
6559		- $roles_table,
6560		- array(
6561		- 'vector_id' => $entry_id,
6562		- 'role_restriction' => $role_restriction,
6563		- 'updated_at' => current_time('mysql')
6564		- ),
6565		- array('%s', '%s', '%s')
6566		- );
6567		-
6568		- // No cache clearing needed since we removed caching
6569		-
6570		- } else {
6571		- // Handle WordPress database role restriction (existing functionality)
6572		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6573		-
6574		- $result = $wpdb->update(
	3270	+ $wpdb->delete(
6575	3271	$table_name,
6576		- array('role_restriction' => $role_restriction),
6577		- array('id' => absint($entry_id)),
6578		- array('%s'),
6579		- array('%d')
6580		- );
6581		- }
6582		-
6583		- if ($result === false) {
6584		- wp_send_json_error('Database update failed: ' . $wpdb->last_error);
6585		- exit;
6586		- }
6587		-
6588		- wp_send_json_success(array(
6589		- 'message' => 'Role restriction updated successfully',
6590		- 'role_restriction' => $role_restriction,
6591		- 'data_source' => $data_source,
6592		- 'entry_id' => $entry_id
6593		- ));
6594		- exit;
6595		-}
6596		-
6597		-// ========================================
6598		-// ROLE-BASED CONTENT RESTRICTIONS - NEW FUNCTIONS
6599		-// Add these to your MxChat_Knowledge_Manager class
6600		-// ========================================
6601		-
6602		-/**
6603		- * Initialize role-based content hooks
6604		- * Add this call to your __construct() or mxchat_init_hooks() method
6605		- */
6606		-private function mxchat_init_role_hooks() {
6607		- // AJAX handlers for tag-role mappings
6608		- add_action('wp_ajax_mxchat_add_tag_role_mapping', array($this, 'ajax_add_tag_role_mapping'));
6609		- add_action('wp_ajax_mxchat_delete_tag_role_mapping', array($this, 'ajax_delete_tag_role_mapping'));
6610		- add_action('wp_ajax_mxchat_get_tag_role_mappings', array($this, 'ajax_get_tag_role_mappings'));
6611		- add_action('wp_ajax_mxchat_bulk_update_tag_roles', array($this, 'ajax_bulk_update_tag_roles'));
6612		-
6613		- // Hook to automatically update role restrictions when tags are added/removed
6614		- add_action('set_object_terms', array($this, 'handle_tag_change'), 10, 6);
6615		-
6616		- // Hook to apply role restrictions on auto-sync
6617		- add_action('mxchat_content_stored', array($this, 'apply_role_restriction_after_storage'), 10, 2);
6618		-}
6619		-
6620		-/**
6621		- * Add tag-role mapping via AJAX
6622		- */
6623		-public function ajax_add_tag_role_mapping() {
6624		- // Verify nonce and permissions
6625		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6626		-
6627		- if (!current_user_can('manage_options')) {
6628		- wp_send_json_error('Unauthorized access');
6629		- exit;
6630		- }
6631		-
6632		- $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
6633		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
6634		-
6635		- if (empty($tag_slug)) {
6636		- wp_send_json_error('Tag slug is required');
6637		- exit;
6638		- }
6639		-
6640		- // Validate role restriction
6641		- $valid_roles = array_keys($this->mxchat_get_role_options());
6642		- if (!in_array($role_restriction, $valid_roles)) {
6643		- wp_send_json_error('Invalid role restriction');
6644		- exit;
6645		- }
6646		-
6647		- // Check if tag exists in WordPress
6648		- $term = get_term_by('slug', $tag_slug, 'post_tag');
6649		- if (!$term) {
6650		- wp_send_json_error('Tag does not exist in WordPress');
6651		- exit;
6652		- }
6653		-
6654		- // Get existing mappings
6655		- $mappings = get_option('mxchat_tag_role_mappings', array());
6656		-
6657		- // Check if mapping already exists
6658		- if (isset($mappings[$tag_slug])) {
6659		- wp_send_json_error('Mapping for this tag already exists');
6660		- exit;
6661		- }
6662		-
6663		- // Add new mapping
6664		- $mappings[$tag_slug] = $role_restriction;
6665		- update_option('mxchat_tag_role_mappings', $mappings);
6666		-
6667		- wp_send_json_success(array(
6668		- 'message' => 'Tag-role mapping added successfully',
6669		- 'tag_slug' => $tag_slug,
6670		- 'role_restriction' => $role_restriction
6671		- ));
6672		- exit;
6673		-}
6674		-
6675		-/**
6676		- * Delete tag-role mapping via AJAX
6677		- */
6678		-public function ajax_delete_tag_role_mapping() {
6679		- // Verify nonce and permissions
6680		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6681		-
6682		- if (!current_user_can('manage_options')) {
6683		- wp_send_json_error('Unauthorized access');
6684		- exit;
6685		- }
6686		-
6687		- $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
6688		-
6689		- if (empty($tag_slug)) {
6690		- wp_send_json_error('Tag slug is required');
6691		- exit;
6692		- }
6693		-
6694		- // Get existing mappings
6695		- $mappings = get_option('mxchat_tag_role_mappings', array());
6696		-
6697		- // Check if mapping exists
6698		- if (!isset($mappings[$tag_slug])) {
6699		- wp_send_json_error('Mapping does not exist');
6700		- exit;
6701		- }
6702		-
6703		- // Remove mapping
6704		- unset($mappings[$tag_slug]);
6705		- update_option('mxchat_tag_role_mappings', $mappings);
6706		-
6707		- wp_send_json_success(array(
6708		- 'message' => 'Tag-role mapping deleted successfully',
6709		- 'tag_slug' => $tag_slug
6710		- ));
6711		- exit;
6712		-}
6713		-
6714		-/**
6715		- * Get all tag-role mappings via AJAX
6716		- */
6717		-public function ajax_get_tag_role_mappings() {
6718		- // Verify nonce and permissions
6719		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6720		-
6721		- if (!current_user_can('manage_options')) {
6722		- wp_send_json_error('Unauthorized access');
6723		- exit;
6724		- }
6725		-
6726		- // Get mappings
6727		- $mappings = get_option('mxchat_tag_role_mappings', array());
6728		- $role_options = $this->mxchat_get_role_options();
6729		-
6730		- $formatted_mappings = array();
6731		-
6732		- foreach ($mappings as $tag_slug => $role_restriction) {
6733		- // Get tag object
6734		- $term = get_term_by('slug', $tag_slug, 'post_tag');
6735		-
6736		- // Count posts with this tag
6737		- $post_count = 0;
6738		- if ($term) {
6739		- $post_count = $term->count;
6740		- }
6741		-
6742		- $formatted_mappings[] = array(
6743		- 'tag_slug' => $tag_slug,
6744		- 'role_restriction' => $role_restriction,
6745		- 'role_label' => $role_options[$role_restriction] ?? $role_restriction,
6746		- 'post_count' => $post_count
6747		- );
6748		- }
6749		-
6750		- wp_send_json_success(array(
6751		- 'mappings' => $formatted_mappings
6752		- ));
6753		- exit;
6754		-}
6755		-
6756		-/**
6757		- * Bulk update role restrictions for all existing content with mapped tags
6758		- */
6759		-public function ajax_bulk_update_tag_roles() {
6760		- // Verify nonce and permissions
6761		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6762		-
6763		- if (!current_user_can('manage_options')) {
6764		- wp_send_json_error('Unauthorized access');
6765		- exit;
6766		- }
6767		-
6768		- // Get mappings
6769		- $mappings = get_option('mxchat_tag_role_mappings', array());
6770		-
6771		- if (empty($mappings)) {
6772		- wp_send_json_error('No tag-role mappings found');
6773		- exit;
6774		- }
6775		-
6776		- global $wpdb;
6777		-
6778		- // Check if using Pinecone
6779		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6780		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6781		-
6782		- $updated_count = 0;
6783		- $details = array();
6784		-
6785		- foreach ($mappings as $tag_slug => $role_restriction) {
6786		- // Get all posts with this tag
6787		- $posts = get_posts(array(
6788		- 'tag' => $tag_slug,
6789		- 'post_type' => 'any',
6790		- 'posts_per_page' => -1,
6791		- 'fields' => 'ids',
6792		- 'post_status' => 'publish'
6793		- ));
6794		-
6795		- if (empty($posts)) {
6796		- continue;
6797		- }
6798		-
6799		- $tag_updated = 0;
6800		-
6801		- foreach ($posts as $post_id) {
6802		- $source_url = get_permalink($post_id);
6803		- if (!$source_url) {
6804		- continue;
6805		- }
6806		-
6807		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
6808		- // Update Pinecone role restriction
6809		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6810		- $vector_id = md5($source_url);
6811		-
6812		- $result = $wpdb->replace(
6813		- $roles_table,
6814		- array(
6815		- 'vector_id' => $vector_id,
6816		- 'role_restriction' => $role_restriction,
6817		- 'updated_at' => current_time('mysql')
6818		- ),
6819		- array('%s', '%s', '%s')
6820		- );
6821		- } else {
6822		- // Update WordPress DB
6823		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6824		-
6825		- $result = $wpdb->update(
6826		- $table_name,
6827		- array('role_restriction' => $role_restriction),
6828		- array('source_url' => $source_url),
6829		- array('%s'),
6830		- array('%s')
6831		- );
6832		- }
6833		-
6834		- if ($result !== false) {
6835		- $tag_updated++;
6836		- $updated_count++;
6837		- }
6838		- }
6839		-
6840		- if ($tag_updated > 0) {
6841		- $details[] = sprintf(
6842		- 'Tag "%s" (%s): %d posts updated',
6843		- $tag_slug,
6844		- $role_restriction,
6845		- $tag_updated
6846		- );
6847		- }
6848		- }
6849		-
6850		- wp_send_json_success(array(
6851		- 'message' => 'Bulk update completed',
6852		- 'updated_count' => $updated_count,
6853		- 'tags_processed' => count($mappings),
6854		- 'details' => $details
6855		- ));
6856		- exit;
6857		-}
6858		-
6859		-/**
6860		- * Handle tag changes on posts (when tags are added or removed)
6861		- */
6862		-public function handle_tag_change($object_id, $terms, $tt_ids, $taxonomy, $append, $old_tt_ids) {
6863		- // Only process post tags
6864		- if ($taxonomy !== 'post_tag') {
6865		- return;
6866		- }
6867		-
6868		- // Get tag-role mappings
6869		- $mappings = get_option('mxchat_tag_role_mappings', array());
6870		-
6871		- if (empty($mappings)) {
6872		- return;
6873		- }
6874		-
6875		- // Get the post's URL
6876		- $source_url = get_permalink($object_id);
6877		- if (!$source_url) {
6878		- return;
6879		- }
6880		-
6881		- // Determine the highest role restriction based on tags
6882		- $highest_role = 'public';
6883		- $role_hierarchy = array(
6884		- 'public' => 0,
6885		- 'logged_in' => 1,
6886		- 'subscriber' => 2,
6887		- 'contributor' => 3,
6888		- 'author' => 4,
6889		- 'editor' => 5,
6890		- 'administrator' => 6
6891		- );
6892		-
6893		- // Get all current tags for the post
6894		- $current_tags = wp_get_post_tags($object_id, array('fields' => 'slugs'));
6895		-
6896		- // Find the highest role restriction among the tags
6897		- foreach ($current_tags as $tag_slug) {
6898		- if (isset($mappings[$tag_slug])) {
6899		- $role = $mappings[$tag_slug];
6900		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
6901		- $highest_role = $role;
6902		- }
6903		- }
6904		- }
6905		-
6906		- // Update the role restriction in the database
6907		- global $wpdb;
6908		-
6909		- // Check if using Pinecone
6910		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6911		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6912		-
6913		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
6914		- // Update Pinecone role restriction
6915		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6916		- $vector_id = md5($source_url);
6917		-
6918		- $wpdb->replace(
6919		- $roles_table,
6920		- array(
6921		- 'vector_id' => $vector_id,
6922		- 'role_restriction' => $highest_role,
6923		- 'updated_at' => current_time('mysql')
6924		- ),
6925		- array('%s', '%s', '%s')
6926		- );
6927		- } else {
6928		- // Update WordPress DB
6929		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6930		-
6931		- $wpdb->update(
6932		- $table_name,
6933		- array('role_restriction' => $highest_role),
6934	3272	array('source_url' => $source_url),
6935		- array('%s'),
6936	3273	array('%s')
6937	3274	);
6938	3275	}
6939	3276	}
6940		-
6941		-/**
6942		- * Apply role restriction after content is stored (for auto-sync)
6943		- */
6944		-public function apply_role_restriction_after_storage($post_id, $source_url) {
6945		- // Get tag-role mappings
6946		- $mappings = get_option('mxchat_tag_role_mappings', array());
6947	3277
6948		- if (empty($mappings)) {
6949		- return;
6950		- }
6951		-
6952		- // Get all tags for the post
6953		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
6954		-
6955		- if (empty($post_tags)) {
6956		- return;
6957		- }
6958		-
6959		- // Determine the highest role restriction based on tags
6960		- $highest_role = 'public';
6961		- $role_hierarchy = array(
6962		- 'public' => 0,
6963		- 'logged_in' => 1,
6964		- 'subscriber' => 2,
6965		- 'contributor' => 3,
6966		- 'author' => 4,
6967		- 'editor' => 5,
6968		- 'administrator' => 6
6969		- );
6970		-
6971		- foreach ($post_tags as $tag_slug) {
6972		- if (isset($mappings[$tag_slug])) {
6973		- $role = $mappings[$tag_slug];
6974		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
6975		- $highest_role = $role;
6976		- }
6977		- }
6978		- }
6979		-
6980		- // If no restricted tags found, return (leave as public)
6981		- if ($highest_role === 'public') {
6982		- return;
6983		- }
6984		-
6985		- // Update the role restriction
6986		- global $wpdb;
6987		-
6988		- // Check if using Pinecone
6989		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6990		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6991		-
6992		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
6993		- // Update Pinecone role restriction
6994		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6995		- $vector_id = md5($source_url);
6996		-
6997		- $wpdb->replace(
6998		- $roles_table,
6999		- array(
7000		- 'vector_id' => $vector_id,
7001		- 'role_restriction' => $highest_role,
7002		- 'updated_at' => current_time('mysql')
7003		- ),
7004		- array('%s', '%s', '%s')
7005		- );
7006		- } else {
7007		- // Update WordPress DB
7008		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7009		-
7010		- $wpdb->update(
7011		- $table_name,
7012		- array('role_restriction' => $highest_role),
7013		- array('source_url' => $source_url),
7014		- array('%s'),
7015		- array('%s')
7016		- );
7017		- }
7018		-}
7019		-
7020		-
7021	3278	// ========================================
7022	3279	// HELPER METHODS
7023	3280	// ========================================
7024	3281
		@@ -7085,1074 +3342,8 @@
7085	3342	*/
7086	3343	private function mxchat_get_pinecone_manager() {
7087	3344	return MxChat_Pinecone_Manager::get_instance();
7088	3345	}
7089		-
7090		-
7091		- // ========================================
7092		-// DATABASE QUEUE TABLE MANAGEMENT
7093		-// ========================================
7094		-
7095		-/**
7096		- * Create queue table on plugin activation
7097		- * Call this from your plugin activation hook
7098		- */
7099		-public function mxchat_create_queue_table() {
7100		- global $wpdb;
7101		-
7102		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7103		- $charset_collate = $wpdb->get_charset_collate();
7104		-
7105		- $sql = "CREATE TABLE IF NOT EXISTS $table_name (
7106		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
7107		- queue_id varchar(64) NOT NULL,
7108		- item_type varchar(20) NOT NULL,
7109		- item_data longtext NOT NULL,
7110		- status varchar(20) NOT NULL DEFAULT 'pending',
7111		- bot_id varchar(50) NOT NULL DEFAULT 'default',
7112		- priority int(11) NOT NULL DEFAULT 0,
7113		- attempts int(11) NOT NULL DEFAULT 0,
7114		- max_attempts int(11) NOT NULL DEFAULT 3,
7115		- error_message text DEFAULT NULL,
7116		- created_at datetime NOT NULL,
7117		- started_at datetime DEFAULT NULL,
7118		- completed_at datetime DEFAULT NULL,
7119		- PRIMARY KEY (id),
7120		- KEY queue_id (queue_id),
7121		- KEY status (status),
7122		- KEY item_type (item_type),
7123		- KEY priority (priority)
7124		- ) $charset_collate;";
7125		-
7126		- require_once(ABSPATH . 'wp-admin/includes/upgrade.php');
7127		- dbDelta($sql);
7128		-
7129		- // Also create a meta table for queue metadata
7130		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7131		-
7132		- $meta_sql = "CREATE TABLE IF NOT EXISTS $meta_table (
7133		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
7134		- queue_id varchar(64) NOT NULL,
7135		- meta_key varchar(255) NOT NULL,
7136		- meta_value longtext,
7137		- PRIMARY KEY (id),
7138		- KEY queue_id (queue_id),
7139		- KEY meta_key (meta_key)
7140		- ) $charset_collate;";
7141		-
7142		- dbDelta($meta_sql);
7143		-}
7144		-
7145		-/**
7146		- * Add items to the processing queue
7147		- *
7148		- * @param string $queue_id Unique identifier for this queue batch
7149		- * @param string $item_type Type of item (url, pdf_page)
7150		- * @param array $items Array of items to queue
7151		- * @param string $bot_id Bot ID for processing
7152		- * @return int Number of items queued
7153		- */
7154		-private function mxchat_add_to_queue($queue_id, $item_type, $items, $bot_id = 'default') {
7155		- global $wpdb;
7156		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7157		-
7158		- $queued_count = 0;
7159		- $priority = 0;
7160		-
7161		- foreach ($items as $item) {
7162		- $result = $wpdb->insert(
7163		- $table_name,
7164		- array(
7165		- 'queue_id' => $queue_id,
7166		- 'item_type' => $item_type,
7167		- 'item_data' => wp_json_encode($item),
7168		- 'status' => 'pending',
7169		- 'bot_id' => $bot_id,
7170		- 'priority' => $priority,
7171		- 'attempts' => 0,
7172		- 'max_attempts' => 3,
7173		- 'created_at' => current_time('mysql')
7174		- ),
7175		- array('%s', '%s', '%s', '%s', '%s', '%d', '%d', '%d', '%s')
7176		- );
7177		-
7178		- if ($result) {
7179		- $queued_count++;
7180		- }
7181		-
7182		- $priority++; // Process in order
7183		- }
7184		-
7185		- return $queued_count;
7186		-}
7187		-
7188		-/**
7189		- * Store queue metadata (total counts, source URL, etc.)
7190		- */
7191		-private function mxchat_set_queue_meta($queue_id, $meta_key, $meta_value) {
7192		- global $wpdb;
7193		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7194		-
7195		- // Check if meta exists
7196		- $existing = $wpdb->get_var($wpdb->prepare(
7197		- "SELECT id FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
7198		- $queue_id,
7199		- $meta_key
7200		- ));
7201		-
7202		- if ($existing) {
7203		- // Update
7204		- $wpdb->update(
7205		- $meta_table,
7206		- array('meta_value' => maybe_serialize($meta_value)),
7207		- array('queue_id' => $queue_id, 'meta_key' => $meta_key),
7208		- array('%s'),
7209		- array('%s', '%s')
7210		- );
7211		- } else {
7212		- // Insert
7213		- $wpdb->insert(
7214		- $meta_table,
7215		- array(
7216		- 'queue_id' => $queue_id,
7217		- 'meta_key' => $meta_key,
7218		- 'meta_value' => maybe_serialize($meta_value)
7219		- ),
7220		- array('%s', '%s', '%s')
7221		- );
7222		- }
7223		-}
7224		-
7225		-/**
7226		- * Get queue metadata
7227		- */
7228		-private function mxchat_get_queue_meta($queue_id, $meta_key) {
7229		- global $wpdb;
7230		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7231		-
7232		- $value = $wpdb->get_var($wpdb->prepare(
7233		- "SELECT meta_value FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
7234		- $queue_id,
7235		- $meta_key
7236		- ));
7237		-
7238		- return maybe_unserialize($value);
7239		-}
7240		-
7241		-// ========================================
7242		-// AJAX QUEUE PROCESSING HANDLERS
7243		-// ========================================
7244		-
7245		-/**
7246		- * AJAX: Get next item from queue to process
7247		- */
7248		-public function ajax_mxchat_get_next_queue_item() {
7249		- // Verify nonce and permissions
7250		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7251		-
7252		- if (!current_user_can('manage_options')) {
7253		- wp_send_json_error('Unauthorized access');
7254		- }
7255		-
7256		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7257		-
7258		- if (empty($queue_id)) {
7259		- wp_send_json_error('Missing queue ID');
7260		- }
7261		-
7262		- global $wpdb;
7263		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7264		-
7265		- // Get next pending item with retry logic for failed items
7266		- $next_item = $wpdb->get_row($wpdb->prepare(
7267		- "SELECT * FROM $table_name
7268		- WHERE queue_id = %s
7269		- AND status IN ('pending', 'failed')
7270		- AND attempts < max_attempts
7271		- ORDER BY priority ASC, id ASC
7272		- LIMIT 1",
7273		- $queue_id
7274		- ));
7275		-
7276		- if (!$next_item) {
7277		- // No more items - queue complete
7278		- wp_send_json_success(array(
7279		- 'complete' => true,
7280		- 'message' => 'Queue processing complete'
7281		- ));
7282		- }
7283		-
7284		- // Mark item as processing
7285		- $wpdb->update(
7286		- $table_name,
7287		- array(
7288		- 'status' => 'processing',
7289		- 'started_at' => current_time('mysql'),
7290		- 'attempts' => $next_item->attempts + 1
7291		- ),
7292		- array('id' => $next_item->id),
7293		- array('%s', '%s', '%d'),
7294		- array('%d')
7295		- );
7296		-
7297		- wp_send_json_success(array(
7298		- 'complete' => false,
7299		- 'item' => array(
7300		- 'id' => $next_item->id,
7301		- 'type' => $next_item->item_type,
7302		- 'data' => json_decode($next_item->item_data, true),
7303		- 'bot_id' => $next_item->bot_id,
7304		- 'attempt' => $next_item->attempts + 1
7305		- )
7306		- ));
7307		-}
7308		-
7309		-/**
7310		- * AJAX: Process a single queue item
7311		- */
7312		-public function ajax_mxchat_process_queue_item() {
7313		- // Verify nonce and permissions
7314		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7315		-
7316		- if (!current_user_can('manage_options')) {
7317		- wp_send_json_error('Unauthorized access');
7318		- }
7319		-
7320		- $item_id = isset($_POST['item_id']) ? absint($_POST['item_id']) : 0;
7321		- $item_type = isset($_POST['item_type']) ? sanitize_text_field($_POST['item_type']) : '';
7322		- $item_data = isset($_POST['item_data']) ? $_POST['item_data'] : array();
7323		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
7324		-
7325		- if (empty($item_id) || empty($item_type)) {
7326		- wp_send_json_error('Missing item data');
7327		- }
7328		-
7329		- global $wpdb;
7330		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7331		-
7332		- // Process based on item type
7333		- try {
7334		- set_time_limit(60); // Give processing 60 seconds
7335		-
7336		- $result = false;
7337		- $error_message = '';
7338		-
7339		- // Read item directly from DB to get queue_id and preserve special chars in item_data
7340		- // (POST round-trip through JS mangles characters like apostrophes in URLs)
7341		- $db_item = $wpdb->get_row($wpdb->prepare(
7342		- "SELECT queue_id, item_data FROM $table_name WHERE id = %d",
7343		- $item_id
7344		- ));
7345		- $item_queue_id = $db_item ? $db_item->queue_id : '';
7346		- if ($db_item && !empty($db_item->item_data)) {
7347		- $db_data = json_decode($db_item->item_data, true);
7348		- if (is_array($db_data)) {
7349		- $item_data = $db_data;
7350		- }
7351		- }
7352		-
7353		- switch ($item_type) {
7354		- case 'url':
7355		- $result = $this->mxchat_process_queue_url($item_data, $bot_id, $item_queue_id);
7356		- break;
7357		-
7358		- case 'pdf_page':
7359		- $result = $this->mxchat_process_queue_pdf_page($item_data, $bot_id);
7360		- break;
7361		-
7362		- default:
7363		- throw new Exception('Unknown item type: ' . $item_type);
7364		- }
7365		-
7366		- if (is_wp_error($result)) {
7367		- $error_code = $result->get_error_code();
7368		- // Content errors (empty page, sanitization) are permanent — retrying won't help
7369		- $permanent_codes = array('empty_page', 'empty_after_sanitization', 'no_api_key', 'page_not_found');
7370		- if (in_array($error_code, $permanent_codes)) {
7371		- // Mark as permanently failed — set attempts = max_attempts so it won't be retried
7372		- $current_item = $wpdb->get_row($wpdb->prepare(
7373		- "SELECT max_attempts FROM $table_name WHERE id = %d", $item_id
7374		- ));
7375		- $wpdb->update(
7376		- $table_name,
7377		- array(
7378		- 'status' => 'failed',
7379		- 'error_message' => $result->get_error_message(),
7380		- 'attempts' => $current_item ? $current_item->max_attempts : 3
7381		- ),
7382		- array('id' => $item_id),
7383		- array('%s', '%s', '%d'),
7384		- array('%d')
7385		- );
7386		- wp_send_json_error(array(
7387		- 'message' => $result->get_error_message(),
7388		- 'permanent_failure' => true,
7389		- 'item_id' => $item_id
7390		- ));
7391		- return;
7392		- }
7393		- throw new Exception($result->get_error_message());
7394		- }
7395		-
7396		- if ($result === false) {
7397		- throw new Exception('Processing returned false - item may be empty or invalid');
7398		- }
7399		-
7400		- // Mark as completed
7401		- $wpdb->update(
7402		- $table_name,
7403		- array(
7404		- 'status' => 'completed',
7405		- 'completed_at' => current_time('mysql'),
7406		- 'error_message' => null
7407		- ),
7408		- array('id' => $item_id),
7409		- array('%s', '%s', '%s'),
7410		- array('%d')
7411		- );
7412		-
7413		- wp_send_json_success(array(
7414		- 'processed' => true,
7415		- 'item_id' => $item_id,
7416		- 'message' => 'Item processed successfully'
7417		- ));
7418		-
7419		- } catch (Exception $e) {
7420		- $error_message = $e->getMessage();
7421		-
7422		- // Get current attempt count
7423		- $item = $wpdb->get_row($wpdb->prepare(
7424		- "SELECT attempts, max_attempts FROM $table_name WHERE id = %d",
7425		- $item_id
7426		- ));
7427		-
7428		- // Check if we've exhausted retries
7429		- if ($item && $item->attempts >= $item->max_attempts) {
7430		- // Permanently failed
7431		- $wpdb->update(
7432		- $table_name,
7433		- array(
7434		- 'status' => 'failed',
7435		- 'error_message' => $error_message
7436		- ),
7437		- array('id' => $item_id),
7438		- array('%s', '%s'),
7439		- array('%d')
7440		- );
7441		-
7442		- wp_send_json_error(array(
7443		- 'message' => 'Item failed after maximum attempts: ' . $error_message,
7444		- 'permanent_failure' => true,
7445		- 'item_id' => $item_id
7446		- ));
7447		- } else {
7448		- // Mark for retry
7449		- $wpdb->update(
7450		- $table_name,
7451		- array(
7452		- 'status' => 'failed',
7453		- 'error_message' => $error_message
7454		- ),
7455		- array('id' => $item_id),
7456		- array('%s', '%s'),
7457		- array('%d')
7458		- );
7459		-
7460		- wp_send_json_error(array(
7461		- 'message' => 'Item processing failed, will retry: ' . $error_message,
7462		- 'can_retry' => true,
7463		- 'item_id' => $item_id,
7464		- 'attempts' => $item ? $item->attempts : 0
7465		- ));
7466		- }
7467		- }
7468		-}
7469		-
7470		-/**
7471		- * Process a URL from the queue
7472		- */
7473		-private function mxchat_process_queue_url($item_data, $bot_id = 'default', $queue_id = '') {
7474		- $url = isset($item_data['url']) ? $item_data['url'] : '';
7475		-
7476		- if (empty($url)) {
7477		- return new WP_Error('invalid_url', 'URL is empty');
7478		- }
7479		-
7480		- // Get bot-specific API key early (needed for both paths)
7481		- $bot_options = $this->get_bot_options($bot_id);
7482		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
7483		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
7484		-
7485		- if (strpos($selected_model, 'voyage') === 0) {
7486		- $api_key = $options['voyage_api_key'] ?? '';
7487		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
7488		- $api_key = $options['gemini_api_key'] ?? '';
7489		- } else {
7490		- $api_key = $options['api_key'] ?? '';
7491		- }
7492		-
7493		- if (empty($api_key)) {
7494		- return new WP_Error('no_api_key', 'API key not configured for bot: ' . $bot_id);
7495		- }
7496		-
7497		- // Check if this is a WooCommerce product URL and WooCommerce is active
7498		- $is_product_url = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
7499		- $content_type = $is_product_url ? 'product' : 'url';
7500		-
7501		- // Try to get WooCommerce product data if it's a product URL
7502		- if ($is_product_url && class_exists('WooCommerce')) {
7503		- $product_content = $this->mxchat_extract_woocommerce_product_content($url);
7504		-
7505		- if (!empty($product_content)) {
7506		- // Successfully extracted WooCommerce product data with pricing
7507		- $result = MxChat_Utils::submit_content_to_db(
7508		- $product_content,
7509		- $url,
7510		- $api_key,
7511		- null,
7512		- $bot_id,
7513		- 'product'
7514		- );
7515		- return $result;
7516		- }
7517		- // If WooCommerce extraction failed, fall through to HTML extraction
7518		- }
7519		-
7520		- // Fetch URL content (fallback for non-products or when WooCommerce extraction fails)
7521		- $is_likely_pdf = (strtolower(pathinfo(parse_url($url, PHP_URL_PATH) ?: '', PATHINFO_EXTENSION)) === 'pdf');
7522		- $response = wp_remote_get($url, array(
7523		- 'timeout' => $is_likely_pdf ? 120 : 30,
7524		- 'redirection' => 5,
7525		- 'user-agent' => 'MxChat/1.0'
7526		- ));
7527		-
7528		- if (is_wp_error($response)) {
7529		- return $response;
7530		- }
7531		-
7532		- $response_code = wp_remote_retrieve_response_code($response);
7533		- if ($response_code !== 200) {
7534		- return new WP_Error('http_error', 'HTTP ' . $response_code . ' error for: ' . $url);
7535		- }
7536		-
7537		- // Check if URL is a PDF — expand into per-page queue items using the standard PDF pipeline
7538		- if ($this->mxchat_is_pdf_url($url, $response)) {
7539		- return $this->mxchat_expand_pdf_to_queue($url, $response, $bot_id, $queue_id);
7540		- }
7541		-
7542		- $html = wp_remote_retrieve_body($response);
7543		-
7544		- if (empty($html)) {
7545		- return new WP_Error('empty_response', 'Empty response body');
7546		- }
7547		-
7548		- // Extract and sanitize content
7549		- $content = $this->mxchat_extract_main_content($html);
7550		- $sanitized = $this->mxchat_sanitize_content_for_api($content);
7551		-
7552		- if (empty($sanitized)) {
7553		- // Not an error - just no content found (maybe a redirect or empty page)
7554		- return false;
7555		- }
7556		-
7557		- // Submit to database with content_type
7558		- $result = MxChat_Utils::submit_content_to_db(
7559		- $sanitized,
7560		- $url,
7561		- $api_key,
7562		- null,
7563		- $bot_id,
7564		- $content_type
7565		- );
7566		-
7567		- return $result;
7568		-}
7569		-
7570		-/**
7571		- * Expand a PDF URL into per-page queue items using the standard PDF pipeline.
7572		- * Called when a sitemap URL turns out to be a PDF — downloads, parses page count,
7573		- * and adds pdf_page items to the same queue so they process with full progress tracking.
7574		- */
7575		-private function mxchat_expand_pdf_to_queue($pdf_url, $response, $bot_id = 'default', $queue_id = '') {
7576		- set_time_limit(120); // PDFs need extra time for download + parsing
7577		-
7578		- $upload_dir = wp_upload_dir();
7579		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
7580		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
7581		-
7582		- $response_body = wp_remote_retrieve_body($response);
7583		- if (empty($response_body)) {
7584		- return new WP_Error('empty_pdf', 'Empty PDF response for: ' . $pdf_url);
7585		- }
7586		-
7587		- if (!wp_mkdir_p(dirname($pdf_path))) {
7588		- return new WP_Error('dir_error', 'Failed to create upload directory');
7589		- }
7590		-
7591		- file_put_contents($pdf_path, $response_body);
7592		-
7593		- if (!file_exists($pdf_path)) {
7594		- return new WP_Error('save_error', 'Failed to save PDF file');
7595		- }
7596		-
7597		- try {
7598		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
7599		-
7600		- if ($total_pages === false || $total_pages < 1) {
7601		- wp_delete_file($pdf_path);
7602		- return new WP_Error('no_pages', 'PDF has no pages: ' . $pdf_url);
7603		- }
7604		-
7605		- // Build per-page items identical to mxchat_handle_pdf_for_knowledge_base
7606		- $pages = array();
7607		- for ($i = 1; $i <= $total_pages; $i++) {
7608		- $pages[] = array(
7609		- 'pdf_path' => $pdf_path,
7610		- 'pdf_url' => $pdf_url,
7611		- 'page_number' => $i,
7612		- 'total_pages' => $total_pages
7613		- );
7614		- }
7615		-
7616		- // Add pdf_page items to the SAME queue so the JS picks them up automatically
7617		- if (!empty($queue_id)) {
7618		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
7619		- } else {
7620		- // Fallback: create a new PDF queue (shouldn't happen in sitemap flow)
7621		- $new_queue_id = 'pdf_' . md5($pdf_url . time());
7622		- $queued_count = $this->mxchat_add_to_queue($new_queue_id, 'pdf_page', $pages, $bot_id);
7623		- $this->mxchat_set_queue_meta($new_queue_id, 'source_url', $pdf_url);
7624		- $this->mxchat_set_queue_meta($new_queue_id, 'queue_type', 'pdf');
7625		- $this->mxchat_set_queue_meta($new_queue_id, 'total_items', $total_pages);
7626		- $this->mxchat_set_queue_meta($new_queue_id, 'bot_id', $bot_id);
7627		- $this->mxchat_set_queue_meta($new_queue_id, 'pdf_path', $pdf_path);
7628		- $this->mxchat_set_queue_meta($new_queue_id, 'created_at', current_time('mysql'));
7629		- }
7630		-
7631		- if ($queued_count === 0) {
7632		- wp_delete_file($pdf_path);
7633		- return new WP_Error('queue_error', 'Failed to add PDF pages to queue');
7634		- }
7635		-
7636		- // Return true so the original URL item is marked complete
7637		- // The new pdf_page items will be processed in subsequent batches
7638		- return true;
7639		-
7640		- } catch (Exception $e) {
7641		- if (file_exists($pdf_path)) {
7642		- wp_delete_file($pdf_path);
7643		- }
7644		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
7645		- }
7646		-}
7647		-
7648		-/**
7649		- * Legacy: Process a PDF URL inline during sitemap queue processing.
7650		- * @deprecated Use mxchat_expand_pdf_to_queue instead — kept for reference only.
7651		- */
7652		-private function mxchat_process_pdf_url_inline($pdf_url, $response, $api_key, $bot_id = 'default') {
7653		- set_time_limit(120); // PDFs need more time — downloading + parsing all pages
7654		-
7655		- $upload_dir = wp_upload_dir();
7656		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
7657		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
7658		-
7659		- $response_body = wp_remote_retrieve_body($response);
7660		- if (empty($response_body)) {
7661		- return new WP_Error('empty_pdf', 'Empty PDF response');
7662		- }
7663		-
7664		- if (!wp_mkdir_p(dirname($pdf_path))) {
7665		- return new WP_Error('dir_error', 'Failed to create upload directory');
7666		- }
7667		-
7668		- file_put_contents($pdf_path, $response_body);
7669		-
7670		- if (!file_exists($pdf_path)) {
7671		- return new WP_Error('save_error', 'Failed to save PDF file');
7672		- }
7673		-
7674		- try {
7675		- mxchat_load_pdf_parser();
7676		- $parser = new \Smalot\PdfParser\Parser();
7677		- $pdf = $parser->parseFile($pdf_path);
7678		- $pages = $pdf->getPages();
7679		- $total_pages = count($pages);
7680		-
7681		- if ($total_pages < 1) {
7682		- wp_delete_file($pdf_path);
7683		- return new WP_Error('no_pages', 'PDF has no pages');
7684		- }
7685		-
7686		- $processed = 0;
7687		- $skipped_pages = array();
7688		-
7689		- for ($i = 0; $i < $total_pages; $i++) {
7690		- $page_num = $i + 1;
7691		- $text = $pages[$i]->getText();
7692		- if (empty($text)) {
7693		- $skipped_pages[] = 'Page ' . $page_num . ': No text could be extracted — page may contain only images, links, or non-standard encoding';
7694		- continue;
7695		- }
7696		-
7697		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
7698		- if (empty($sanitized)) {
7699		- $skipped_pages[] = 'Page ' . $page_num . ': Text was extracted but contained only special characters, control codes, or unsupported content';
7700		- continue;
7701		- }
7702		-
7703		- $metadata = array(
7704		- 'document_type' => 'pdf',
7705		- 'total_pages' => $total_pages,
7706		- 'current_page' => $page_num,
7707		- 'source_url' => $pdf_url,
7708		- );
7709		-
7710		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
7711		- $page_url = esc_url($pdf_url . '#page=' . $page_num);
7712		-
7713		- MxChat_Utils::submit_content_to_db(
7714		- $content_with_metadata,
7715		- $page_url,
7716		- $api_key,
7717		- null,
7718		- $bot_id,
7719		- 'pdf'
7720		- );
7721		-
7722		- $processed++;
7723		- }
7724		-
7725		- // Clean up the temp PDF file
7726		- wp_delete_file($pdf_path);
7727		-
7728		- if (!empty($skipped_pages)) {
7729		- error_log('MxChat PDF: Skipped ' . count($skipped_pages) . ' of ' . $total_pages . ' pages: ' . implode('; ', $skipped_pages));
7730		- }
7731		-
7732		- return $processed > 0 ? true : false;
7733		-
7734		- } catch (Exception $e) {
7735		- if (file_exists($pdf_path)) {
7736		- wp_delete_file($pdf_path);
7737		- }
7738		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
7739		- }
7740		-}
7741		-
7742		-/**
7743		- * Extract WooCommerce product content including pricing
7744		- *
7745		- * @param string $url The product URL
7746		- * @return string|false Product content with pricing, or false if not found
7747		- */
7748		-private function mxchat_extract_woocommerce_product_content($url) {
7749		- // Try to get product ID from URL
7750		- $product_id = url_to_postid($url);
7751		-
7752		- // If url_to_postid fails, try to extract from URL pattern
7753		- if (!$product_id) {
7754		- $product_slug = '';
7755		-
7756		- // Handle pretty permalinks: /product/product-name/
7757		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
7758		- $product_slug = $matches[1];
7759		- }
7760		-
7761		- if (!empty($product_slug)) {
7762		- $product_post = get_page_by_path($product_slug, OBJECT, 'product');
7763		- if ($product_post) {
7764		- $product_id = $product_post->ID;
7765		- }
7766		- }
7767		- }
7768		-
7769		- if (!$product_id) {
7770		- return false;
7771		- }
7772		-
7773		- // Get WooCommerce product object
7774		- $product = wc_get_product($product_id);
7775		-
7776		- if (!$product) {
7777		- return false;
7778		- }
7779		-
7780		- // Build product content with pricing (similar to mxchat_store_product_embedding)
7781		- $title = $product->get_name();
7782		- $description = $product->get_description();
7783		- $short_description = $product->get_short_description();
7784		- $sku = $product->get_sku();
7785		-
7786		- // Get pricing information
7787		- $regular_price = $product->get_regular_price();
7788		- $sale_price = $product->get_sale_price();
7789		- $price = $product->get_price(); // Current active price
7790		-
7791		- // Get currency symbol
7792		- $currency_symbol = get_woocommerce_currency_symbol();
7793		-
7794		- // Format content
7795		- $content = $title . "\n\n";
7796		-
7797		- if (!empty($short_description)) {
7798		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
7799		- }
7800		-
7801		- if (!empty($description)) {
7802		- $content .= wp_strip_all_tags($description) . "\n\n";
7803		- }
7804		-
7805		- // Add pricing information
7806		- if (!empty($regular_price)) {
7807		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
7808		- } elseif (!empty($price)) {
7809		- $content .= "Price: " . $currency_symbol . $price . "\n";
7810		- }
7811		-
7812		- if (!empty($sale_price) && $sale_price !== $regular_price) {
7813		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
7814		- }
7815		-
7816		- // Handle variable products - show price range
7817		- if ($product->is_type('variable')) {
7818		- $min_price = $product->get_variation_price('min');
7819		- $max_price = $product->get_variation_price('max');
7820		- if ($min_price !== $max_price) {
7821		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
7822		- }
7823		- }
7824		-
7825		- if (!empty($sku)) {
7826		- $content .= "SKU: " . $sku . "\n";
7827		- }
7828		-
7829		- // Get product categories
7830		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
7831		- if (!empty($categories) && !is_wp_error($categories)) {
7832		- $content .= "Categories: " . implode(', ', $categories) . "\n";
7833		- }
7834		-
7835		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
7836		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
7837		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
7838		- foreach ($custom_tabs as $tab) {
7839		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
7840		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
7841		-
7842		- if (!empty($tab_title) && !empty($tab_content)) {
7843		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
7844		- }
7845		- }
7846		- }
7847		-
7848		- // Also check for reusable/saved tabs applied to this product
7849		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
7850		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
7851		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
7852		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
7853		- foreach ($applied_saved_tabs as $saved_tab_id) {
7854		- if (isset($saved_tabs[$saved_tab_id])) {
7855		- $tab = $saved_tabs[$saved_tab_id];
7856		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
7857		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
7858		-
7859		- if (!empty($tab_title) && !empty($tab_content)) {
7860		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
7861		- }
7862		- }
7863		- }
7864		- }
7865		- }
7866		-
7867		- return $this->mxchat_sanitize_content_for_api($content);
7868		-}
7869		-
7870		-/**
7871		- * Process a PDF page from the queue
7872		- */
7873		-private function mxchat_process_queue_pdf_page($item_data, $bot_id = 'default') {
7874		- $pdf_path = isset($item_data['pdf_path']) ? $item_data['pdf_path'] : '';
7875		- $pdf_url = isset($item_data['pdf_url']) ? $item_data['pdf_url'] : '';
7876		- $page_number = isset($item_data['page_number']) ? absint($item_data['page_number']) : 0;
7877		- $total_pages = isset($item_data['total_pages']) ? absint($item_data['total_pages']) : 0;
7878		-
7879		- if (empty($pdf_path) || !file_exists($pdf_path)) {
7880		- return new WP_Error('pdf_not_found', 'PDF file not found: ' . $pdf_path);
7881		- }
7882		-
7883		- if ($page_number < 1) {
7884		- return new WP_Error('invalid_page', 'Invalid page number');
7885		- }
7886		-
7887		- try {
7888		- mxchat_load_pdf_parser();
7889		- $parser = new \Smalot\PdfParser\Parser();
7890		- $pdf = $parser->parseFile($pdf_path);
7891		- $pages = $pdf->getPages();
7892		-
7893		- if (!isset($pages[$page_number - 1])) {
7894		- return new WP_Error('page_not_found', 'Page ' . $page_number . ' not found in PDF');
7895		- }
7896		-
7897		- $text = $pages[$page_number - 1]->getText();
7898		-
7899		- if (empty($text)) {
7900		- return new WP_Error('empty_page', 'Page ' . $page_number . ': No text could be extracted — page may contain only images, links, or non-standard encoding');
7901		- }
7902		-
7903		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
7904		-
7905		- if (empty($sanitized)) {
7906		- return new WP_Error('empty_after_sanitization', 'Page ' . $page_number . ': Text was extracted but contained only special characters, control codes, or unsupported content that was removed during cleanup');
7907		- }
7908		-
7909		- // Create metadata
7910		- $metadata = array(
7911		- 'document_type' => 'pdf',
7912		- 'total_pages' => $total_pages,
7913		- 'current_page' => $page_number,
7914		- 'source_url' => $pdf_url
7915		- );
7916		-
7917		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
7918		- $page_url = esc_url($pdf_url . "#page=" . $page_number);
7919		-
7920		- // Get bot-specific API key
7921		- $bot_options = $this->get_bot_options($bot_id);
7922		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
7923		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
7924		-
7925		- if (strpos($selected_model, 'voyage') === 0) {
7926		- $api_key = $options['voyage_api_key'] ?? '';
7927		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
7928		- $api_key = $options['gemini_api_key'] ?? '';
7929		- } else {
7930		- $api_key = $options['api_key'] ?? '';
7931		- }
7932		-
7933		- if (empty($api_key)) {
7934		- return new WP_Error('no_api_key', 'API key not configured for bot: ' . $bot_id);
7935		- }
7936		-
7937		- // Submit to database - UPDATED 2.5.6: Added content_type 'pdf'
7938		- $result = MxChat_Utils::submit_content_to_db(
7939		- $content_with_metadata,
7940		- $page_url,
7941		- $api_key,
7942		- null,
7943		- $bot_id,
7944		- 'pdf'
7945		- );
7946		-
7947		- return $result;
7948		-
7949		- } catch (Exception $e) {
7950		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
7951		- }
7952		-}
7953		-
7954		-/**
7955		- * AJAX: Get queue processing status
7956		- */
7957		-public function ajax_mxchat_get_queue_status() {
7958		- // Verify nonce and permissions
7959		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7960		-
7961		- if (!current_user_can('manage_options')) {
7962		- wp_send_json_error('Unauthorized access');
7963		- }
7964		-
7965		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7966		-
7967		- if (empty($queue_id)) {
7968		- wp_send_json_error('Missing queue ID');
7969		- }
7970		-
7971		- global $wpdb;
7972		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7973		-
7974		- // Get counts by status
7975		- $counts = $wpdb->get_results($wpdb->prepare(
7976		- "SELECT status, COUNT(*) as count
7977		- FROM $table_name
7978		- WHERE queue_id = %s
7979		- GROUP BY status",
7980		- $queue_id
7981		- ), OBJECT_K);
7982		-
7983		- $total = 0;
7984		- $completed = 0;
7985		- $failed = 0;
7986		- $processing = 0;
7987		- $pending = 0;
7988		-
7989		- foreach ($counts as $status => $data) {
7990		- $count = absint($data->count);
7991		- $total += $count;
7992		-
7993		- switch ($status) {
7994		- case 'completed':
7995		- $completed = $count;
7996		- break;
7997		- case 'failed':
7998		- $failed = $count;
7999		- break;
8000		- case 'processing':
8001		- $processing = $count;
8002		- break;
8003		- case 'pending':
8004		- $pending = $count;
8005		- break;
8006		- }
8007		- }
8008		-
8009		- // Calculate percentage
8010		- $percentage = $total > 0 ? round((($completed + $failed) / $total) * 100) : 0;
8011		-
8012		- // Get failed items details (include all failed items, not just those that exhausted retries)
8013		- $failed_items = array();
8014		- if ($failed > 0) {
8015		- $failed_items = $wpdb->get_results($wpdb->prepare(
8016		- "SELECT item_type, item_data, error_message, attempts
8017		- FROM $table_name
8018		- WHERE queue_id = %s
8019		- AND status = 'failed'
8020		- ORDER BY id DESC
8021		- LIMIT 50",
8022		- $queue_id
8023		- ));
8024		- }
8025		-
8026		- // Get queue metadata
8027		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
8028		- $queue_type = $this->mxchat_get_queue_meta($queue_id, 'queue_type');
8029		-
8030		- // Determine if queue is complete
8031		- $is_complete = ($pending === 0 && $processing === 0);
8032		-
8033		- wp_send_json_success(array(
8034		- 'queue_id' => $queue_id,
8035		- 'queue_type' => $queue_type,
8036		- 'source_url' => $source_url,
8037		- 'total' => $total,
8038		- 'completed' => $completed,
8039		- 'failed' => $failed,
8040		- 'processing' => $processing,
8041		- 'pending' => $pending,
8042		- 'percentage' => $percentage,
8043		- 'is_complete' => $is_complete,
8044		- 'failed_items' => $failed_items,
8045		- 'status' => $is_complete ? 'complete' : 'processing'
8046		- ));
8047		-}
8048		-
8049		-/**
8050		- * AJAX: Clear completed queue
8051		- */
8052		-public function ajax_mxchat_clear_queue() {
8053		- // Verify nonce and permissions
8054		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8055		-
8056		- if (!current_user_can('manage_options')) {
8057		- wp_send_json_error('Unauthorized access');
8058		- }
8059		-
8060		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8061		-
8062		- if (empty($queue_id)) {
8063		- wp_send_json_error('Missing queue ID');
8064		- }
8065		-
8066		- global $wpdb;
8067		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8068		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
8069		-
8070		- // Delete queue items
8071		- $wpdb->delete(
8072		- $table_name,
8073		- array('queue_id' => $queue_id),
8074		- array('%s')
8075		- );
8076		-
8077		- // Delete queue metadata
8078		- $wpdb->delete(
8079		- $meta_table,
8080		- array('queue_id' => $queue_id),
8081		- array('%s')
8082		- );
8083		-
8084		- wp_send_json_success(array(
8085		- 'message' => 'Queue cleared successfully'
8086		- ));
8087		-}
8088		-
8089		-/**
8090		- * AJAX: Retry failed items in queue
8091		- */
8092		-public function ajax_mxchat_retry_failed() {
8093		- // Verify nonce and permissions
8094		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8095		-
8096		- if (!current_user_can('manage_options')) {
8097		- wp_send_json_error('Unauthorized access');
8098		- }
8099		-
8100		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8101		-
8102		- if (empty($queue_id)) {
8103		- wp_send_json_error('Missing queue ID');
8104		- }
8105		-
8106		- global $wpdb;
8107		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8108		-
8109		- // Reset failed items to pending and reset attempt count
8110		- $updated = $wpdb->update(
8111		- $table_name,
8112		- array(
8113		- 'status' => 'pending',
8114		- 'attempts' => 0,
8115		- 'error_message' => null
8116		- ),
8117		- array(
8118		- 'queue_id' => $queue_id,
8119		- 'status' => 'failed'
8120		- ),
8121		- array('%s', '%d', '%s'),
8122		- array('%s', '%s')
8123		- );
8124		-
8125		- wp_send_json_success(array(
8126		- 'message' => 'Reset ' . $updated . ' failed items for retry',
8127		- 'reset_count' => $updated
8128		- ));
8129		-}
8130		-
8131		-
8132		-public function ajax_mxchat_mark_queue_complete() {
8133		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8134		-
8135		- if (!current_user_can('manage_options')) {
8136		- wp_send_json_error('Unauthorized access');
8137		- }
8138		-
8139		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8140		-
8141		- if (empty($queue_id)) {
8142		- wp_send_json_error('Missing queue ID');
8143		- }
8144		-
8145		- // Clear active queue transients
8146		- if (strpos($queue_id, 'sitemap_') === 0) {
8147		- delete_transient('mxchat_active_queue_sitemap');
8148		- } else if (strpos($queue_id, 'pdf_') === 0) {
8149		- delete_transient('mxchat_active_queue_pdf');
8150		- }
8151		-
8152		- wp_send_json_success(array('message' => 'Queue marked as complete'));
8153		-}
8154		-
8155	3346
8156	3347	// ========================================
8157	3348	// STATIC ACCESS METHODS
8158	3349	// ========================================