← All changes | admin/class-knowledge-manager.php +1560 -6127 3.2.1 → 2.2.3 View file →

		@@ -15,75 +15,49 @@
15	15
16	16	/**
17	17	* Constructor - Register hooks for content processing
18	18	*/
19		-public function __construct() {
20		- $this->options = get_option('mxchat_options', array());
21		- $this->mxchat_init_hooks();
	19	+ public function __construct() {
	20	+ $this->options = get_option('mxchat_options', array());
	21	+ $this->mxchat_init_hooks();
	22	+ }
22	23
23		- $this->mxchat_init_role_hooks();
24		-}
25		-
26		-/**
27		- * Initialize WordPress hooks for content processing
28		- *
29		- */
30		-private function mxchat_init_hooks() {
31		- // Admin post handlers for form submissions
32		- add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
33		- add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
34		- add_action('admin_post_mxchat_submit_pdf_file', array($this, 'mxchat_handle_pdf_file_submission'));
35		- add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
36		-
37		- // AJAX handlers for real-time processing and status updates
38		- add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
39		- add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status'));
40		- add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
41		- add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
42		- add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
43		- add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
44		- add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
45		- add_action('wp_ajax_mxchat_delete_chunks_by_url', array($this, 'ajax_mxchat_delete_chunks_by_url'));
46		- add_action('wp_ajax_mxchat_delete_wordpress_prompt', array($this, 'ajax_mxchat_delete_wordpress_prompt'));
47		- add_action('wp_ajax_mxchat_bulk_delete_knowledge', array($this, 'ajax_mxchat_bulk_delete_knowledge'));
48		- add_action('wp_ajax_mxchat_update_role_restriction', array($this, 'ajax_mxchat_update_role_restriction'));
49		-
50		- // Queue-based processing AJAX handlers
51		- add_action('wp_ajax_mxchat_get_next_queue_item', array($this, 'ajax_mxchat_get_next_queue_item'));
52		- add_action('wp_ajax_mxchat_process_queue_item', array($this, 'ajax_mxchat_process_queue_item'));
53		- add_action('wp_ajax_mxchat_get_queue_status', array($this, 'ajax_mxchat_get_queue_status'));
54		- add_action('wp_ajax_mxchat_clear_queue', array($this, 'ajax_mxchat_clear_queue'));
55		- add_action('wp_ajax_mxchat_retry_failed', array($this, 'ajax_mxchat_retry_failed'));
56		- add_action('wp_ajax_mxchat_get_recent_entries', array($this, 'ajax_mxchat_get_recent_entries'));
57		- add_action('wp_ajax_mxchat_detect_sitemaps', array($this, 'ajax_mxchat_detect_sitemaps'));
58		- add_action('wp_ajax_mxchat_refresh_pinecone_entries', array($this, 'ajax_mxchat_refresh_pinecone_entries'));
59		- add_action('wp_ajax_mxchat_paginate_entries', array($this, 'ajax_mxchat_paginate_entries'));
60		- add_action('wp_ajax_mxchat_get_entry_content', array($this, 'ajax_mxchat_get_entry_content'));
61		- add_action('wp_ajax_mxchat_save_entry_content', array($this, 'ajax_mxchat_save_entry_content'));
	24	+ /**
	25	+ * Initialize WordPress hooks for content processing
	26	+ */
	27	+ private function mxchat_init_hooks() {
	28	+ // Admin post handlers for form submissions
	29	+ add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
	30	+ add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
	31	+ add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
	32	+
	33	+ // AJAX handlers for real-time processing and status updates
	34	+ add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
	35	+ add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status')); // NEW
	36	+ add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
	37	+ add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
	38	+ add_action('wp_ajax_mxchat_manual_batch_process', array($this, 'ajax_manual_batch_process'));
	39	+ add_action('mxchat_delete_content', array($this, 'mxchat_delete_from_pinecone_by_url'), 10, 1);
62	40
63		- // Hook for content deletion
64		- add_action('mxchat_delete_content', array($this, 'mxchat_delete_from_pinecone_by_url'), 10, 1);
65		-
66		- // WordPress post management hooks
67		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
68		- add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
69		- add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
70		- add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
	41	+
	42	+ // Cron handlers for background processing
	43	+ add_action('mxchat_process_sitemap_urls', array($this, 'mxchat_process_sitemap_urls_cron'), 10, 5);
	44	+ add_action('mxchat_process_pdf_pages', array($this, 'mxchat_process_pdf_pages_cron'), 10, 5);
	45	+
	46	+ // WordPress post management hooks
	47	+ add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
	48	+ add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
	49	+ add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
	50	+ add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
71	51
72		- // ACF hook - fires AFTER ACF fields are saved, ensuring ACF data is available
73		- // Priority 20 to run after ACF's own save (which runs at priority 10)
74		- add_action('acf/save_post', array($this, 'mxchat_handle_acf_save'), 20);
75		-
76		- add_action('wp_ajax_mxchat_mark_queue_complete', array($this, 'ajax_mxchat_mark_queue_complete'));
77		-
78		- // WooCommerce product hooks (if WooCommerce is active)
79		- if (class_exists('WooCommerce')) {
80		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
81		- add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
82		- add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
83		- add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
	52	+ // WooCommerce product hooks (if WooCommerce is active)
	53	+ if (class_exists('WooCommerce')) {
	54	+ add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
	55	+ add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
	56	+ add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
	57	+ }
	58	+
84	59	}
85		-}
86	60
87	61	/**
88	62	* Get current options (refreshed)
89	63	*/
		@@ -93,9 +67,315 @@
93	67	}
94	68	return $this->options;
95	69	}
96	70
	71	+
	72	+ /**
	73	+ * Handle manual batch processing via AJAX
	74	+ */
	75	+public function ajax_manual_batch_process() {
	76	+ try {
	77	+ // Verify nonce and permissions
	78	+ check_ajax_referer('mxchat_status_nonce', 'nonce');
	79	+
	80	+ if (!current_user_can('manage_options')) {
	81	+ wp_send_json_error('Unauthorized access');
	82	+ }
	83	+
	84	+ $process_type = sanitize_text_field($_POST['process_type'] ?? '');
	85	+ $url = sanitize_text_field($_POST['url'] ?? '');
	86	+
	87	+ if (empty($process_type) || empty($url)) {
	88	+ wp_send_json_error('Missing required parameters');
	89	+ }
	90	+
	91	+ $processed = 0;
	92	+
	93	+ if ($process_type === 'pdf') {
	94	+ $processed = $this->mxchat_manual_process_pdf_batch($url);
	95	+ } elseif ($process_type === 'sitemap') {
	96	+ $processed = $this->mxchat_manual_process_sitemap_batch($url);
	97	+ }
	98	+
	99	+ if ($processed > 0) {
	100	+ wp_send_json_success(array(
	101	+ 'message' => "Processed {$processed} items successfully",
	102	+ 'processed' => $processed
	103	+ ));
	104	+ } else {
	105	+ wp_send_json_error('No items were processed');
	106	+ }
	107	+
	108	+ } catch (Exception $e) {
	109	+ //error_log('Manual batch process error: ' . $e->getMessage());
	110	+ wp_send_json_error('Processing failed: ' . $e->getMessage());
	111	+ }
	112	+}
97	113
	114	+/**
	115	+ * Process a small PDF batch manually - DIRECT PROCESSING
	116	+ */
	117	+private function mxchat_manual_process_pdf_batch($pdf_url) {
	118	+ try {
	119	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	120	+ $status = get_transient($status_key);
	121	+
	122	+ if (!$status || $status['status'] !== 'processing') {
	123	+ //error_log('Manual PDF: No processing status found');
	124	+ return 0;
	125	+ }
	126	+
	127	+ //error_log('Manual PDF: Starting direct processing for ' . $pdf_url);
	128	+
	129	+ // Get current progress
	130	+ $current_page = $status['processed_pages'] ?? 0;
	131	+ $total_pages = $status['total_pages'] ?? 0;
	132	+
	133	+ if ($current_page >= $total_pages) {
	134	+ //error_log('Manual PDF: Already completed');
	135	+ return 0;
	136	+ }
	137	+
	138	+ // Try to download the PDF again for processing
	139	+ $response = wp_remote_get($pdf_url, array('timeout' => 30));
	140	+
	141	+ if (is_wp_error($response)) {
	142	+ //error_log('Manual PDF: Failed to download PDF: ' . $response->get_error_message());
	143	+ return 0;
	144	+ }
	145	+
	146	+ $pdf_content = wp_remote_retrieve_body($response);
	147	+ if (empty($pdf_content)) {
	148	+ //error_log('Manual PDF: Empty PDF content');
	149	+ return 0;
	150	+ }
	151	+
	152	+ // Save PDF temporarily
	153	+ $upload_dir = wp_upload_dir();
	154	+ $temp_pdf_path = trailingslashit($upload_dir['path']) . 'temp_manual_' . time() . '.pdf';
	155	+ file_put_contents($temp_pdf_path, $pdf_content);
	156	+
	157	+ // Process 2 pages directly
	158	+ $processed = $this->mxchat_process_pdf_pages_direct($temp_pdf_path, $pdf_url, $current_page, 5);
	159	+
	160	+ // Clean up temp file
	161	+ if (file_exists($temp_pdf_path)) {
	162	+ wp_delete_file($temp_pdf_path);
	163	+ }
	164	+
	165	+ //error_log('Manual PDF: Processed ' . $processed . ' pages');
	166	+ return $processed;
	167	+
	168	+ } catch (Exception $e) {
	169	+ //error_log('Manual PDF batch error: ' . $e->getMessage());
	170	+ return 0;
	171	+ }
	172	+}
	173	+
	174	+/**
	175	+ * Process PDF pages directly without cron
	176	+ */
	177	+private function mxchat_process_pdf_pages_direct($pdf_path, $pdf_url, $start_page, $batch_size) {
	178	+ try {
	179	+ if (!file_exists($pdf_path)) {
	180	+ //error_log('Direct PDF: File not found at ' . $pdf_path);
	181	+ return 0;
	182	+ }
	183	+
	184	+ $parser = new \Smalot\PdfParser\Parser();
	185	+ $pdf = $parser->parseFile($pdf_path);
	186	+ $pages = $pdf->getPages();
	187	+
	188	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	189	+ $status = get_transient($status_key);
	190	+
	191	+ if (!$status) {
	192	+ return 0;
	193	+ }
	194	+
	195	+ $options = get_option('mxchat_options');
	196	+ $api_key = $options['api_key'] ?? '';
	197	+
	198	+ if (empty($api_key)) {
	199	+ //error_log('Direct PDF: No API key');
	200	+ return 0;
	201	+ }
	202	+
	203	+ $processed = 0;
	204	+ $end_page = min($start_page + $batch_size, count($pages));
	205	+
	206	+ for ($i = $start_page; $i < $end_page; $i++) {
	207	+ try {
	208	+ $page_number = $i + 1;
	209	+ $text = $pages[$i]->getText();
	210	+
	211	+ if (empty($text)) {
	212	+ //error_log('Direct PDF: Empty text on page ' . $page_number);
	213	+ continue;
	214	+ }
	215	+
	216	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
	217	+ if (empty($sanitized_content)) {
	218	+ //error_log('Direct PDF: No content after sanitization on page ' . $page_number);
	219	+ continue;
	220	+ }
	221	+
	222	+ // Generate embedding
	223	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	224	+ if (is_string($embedding_vector)) {
	225	+ //error_log('Direct PDF: Embedding failed on page ' . $page_number . ': ' . $embedding_vector);
	226	+ continue;
	227	+ }
	228	+
	229	+ // Create metadata
	230	+ $metadata = array(
	231	+ 'document_type' => 'pdf',
	232	+ 'total_pages' => count($pages),
	233	+ 'current_page' => $page_number,
	234	+ 'source_url' => $pdf_url
	235	+ );
	236	+
	237	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	238	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
	239	+
	240	+ // Store in database
	241	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $api_key);
	242	+
	243	+ if (is_wp_error($db_result)) {
	244	+ //error_log('Direct PDF: DB error on page ' . $page_number . ': ' . $db_result->get_error_message());
	245	+ continue;
	246	+ }
	247	+
	248	+ $processed++;
	249	+ //error_log('Direct PDF: Successfully processed page ' . $page_number);
	250	+
	251	+ // Update status
	252	+ $status['processed_pages'] = $i + 1;
	253	+ $status['last_update'] = time();
	254	+ $status['percentage'] = round(($status['processed_pages'] / $status['total_pages']) * 100);
	255	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	256	+
	257	+ } catch (Exception $e) {
	258	+ //error_log('Direct PDF: Error processing page ' . ($i + 1) . ': ' . $e->getMessage());
	259	+ continue;
	260	+ }
	261	+ }
	262	+
	263	+ // Check if completed
	264	+ if ($status['processed_pages'] >= $status['total_pages']) {
	265	+ $status['status'] = 'complete';
	266	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	267	+ //error_log('Direct PDF: Processing completed');
	268	+ }
	269	+
	270	+ return $processed;
	271	+
	272	+ } catch (Exception $e) {
	273	+ //error_log('Direct PDF processing error: ' . $e->getMessage());
	274	+ return 0;
	275	+ }
	276	+}
	277	+
	278	+/**
	279	+ * Process a small sitemap batch manually - DIRECT PROCESSING
	280	+ */
	281	+private function mxchat_manual_process_sitemap_batch($sitemap_url) {
	282	+ try {
	283	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	284	+ $status = get_transient($status_key);
	285	+
	286	+ if (!$status || $status['status'] !== 'processing') {
	287	+ return 0;
	288	+ }
	289	+
	290	+ //error_log('Manual Sitemap: Starting direct processing for ' . $sitemap_url);
	291	+
	292	+ // Re-fetch the sitemap to get URLs
	293	+ $response = wp_remote_get($sitemap_url, array('timeout' => 30));
	294	+ if (is_wp_error($response)) {
	295	+ //error_log('Manual Sitemap: Failed to fetch sitemap');
	296	+ return 0;
	297	+ }
	298	+
	299	+ $sitemap_content = wp_remote_retrieve_body($response);
	300	+ $xml = simplexml_load_string($sitemap_content);
	301	+
	302	+ if (!$xml) {
	303	+ //error_log('Manual Sitemap: Invalid XML');
	304	+ return 0;
	305	+ }
	306	+
	307	+ $urls = array();
	308	+ foreach ($xml->url as $url_element) {
	309	+ $urls[] = (string)$url_element->loc;
	310	+ }
	311	+
	312	+ $current_processed = $status['processed_urls'] ?? 0;
	313	+ $batch_size = 5;
	314	+ $processed = 0;
	315	+
	316	+ // Process next 2 URLs
	317	+ for ($i = $current_processed; $i < min($current_processed + $batch_size, count($urls)); $i++) {
	318	+ $url = $urls[$i];
	319	+
	320	+ if ($this->mxchat_process_single_url_direct($url)) {
	321	+ $processed++;
	322	+ }
	323	+
	324	+ // Update status
	325	+ $status['processed_urls'] = $i + 1;
	326	+ $status['last_update'] = time();
	327	+ $status['percentage'] = round(($status['processed_urls'] / $status['total_urls']) * 100);
	328	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	329	+ }
	330	+
	331	+ // Check if completed
	332	+ if ($status['processed_urls'] >= $status['total_urls']) {
	333	+ $status['status'] = 'complete';
	334	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	335	+ }
	336	+
	337	+ //error_log('Manual Sitemap: Processed ' . $processed . ' URLs');
	338	+ return $processed;
	339	+
	340	+ } catch (Exception $e) {
	341	+ //error_log('Manual sitemap batch error: ' . $e->getMessage());
	342	+ return 0;
	343	+ }
	344	+}
	345	+
	346	+/**
	347	+ * Process a single URL directly
	348	+ */
	349	+private function mxchat_process_single_url_direct($url) {
	350	+ try {
	351	+ $response = wp_remote_get($url, array('timeout' => 30));
	352	+ if (is_wp_error($response)) {
	353	+ return false;
	354	+ }
	355	+
	356	+ $html = wp_remote_retrieve_body($response);
	357	+ $content = $this->mxchat_extract_main_content($html);
	358	+ $sanitized = $this->mxchat_sanitize_content_for_api($content);
	359	+
	360	+ if (empty($sanitized)) {
	361	+ return false;
	362	+ }
	363	+
	364	+ $options = get_option('mxchat_options');
	365	+ $api_key = $options['api_key'] ?? '';
	366	+
	367	+ $result = MxChat_Utils::submit_content_to_db($sanitized, $url, $api_key);
	368	+
	369	+ return !is_wp_error($result);
	370	+
	371	+ } catch (Exception $e) {
	372	+ //error_log('Single URL processing error: ' . $e->getMessage());
	373	+ return false;
	374	+ }
	375	+}
	376	+
	377	+
98	378	// ========================================
99	379	// MAIN CONTENT SUBMISSION HANDLERS
100	380	// ========================================
101	381
		@@ -111,18 +391,13 @@
111	391	wp_die(esc_html__('Nonce verification failed.', 'mxchat'));
112	392	}
113	393
114	394	// Sanitize the inputs.
115		- // Use wp_kses_post to allow safe HTML and wp_unslash to remove WordPress added slashes
116		- $article_content = wp_kses_post(wp_unslash($_POST['article_content']));
	395	+ $article_content = sanitize_textarea_field($_POST['article_content']);
117	396	$article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
118	397
119		- // Get bot_id from form submission
120		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
121		-
122		- // Get bot-specific options and API key
123		- $bot_options = $this->get_bot_options($bot_id);
124		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	398	+ // Get API key for submission
	399	+ $options = get_option('mxchat_options');
125	400	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
126	401
127	402	if (strpos($selected_model, 'voyage') === 0) {
128	403	$api_key = $options['voyage_api_key'] ?? '';
		@@ -140,10 +415,10 @@
140	415	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
141	416	exit;
142	417	}
143	418
144		- // Use centralized utility function with bot_id
145		- $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key, null, $bot_id);
	419	+ // Use centralized utility function for storage
	420	+ $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key);
146	421
147	422	if (is_wp_error($result)) {
148	423	set_transient('mxchat_admin_notice_error',
149	424	esc_html__('Error storing content: ', 'mxchat') . $result->get_error_message(),
		@@ -158,23 +433,17 @@
158	433
159	434	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
160	435	exit;
161	436	}
162		-
163	437	public function mxchat_is_pdf_url($url, $response) {
164	438	$content_type = wp_remote_retrieve_header($response, 'content-type');
165	439	$file_extension = strtolower(pathinfo($url, PATHINFO_EXTENSION));
166	440
167		- // Check Content-Disposition header for .pdf filename (Google Drive sends this)
168		- $disposition = wp_remote_retrieve_header($response, 'content-disposition');
169		- $has_pdf_disposition = ! empty($disposition) && stripos($disposition, '.pdf') !== false;
170		-
171		- return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf' || $has_pdf_disposition;
	441	+ return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf';
172	442	}
173		-
174		-
175		-public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response, $bot_id = 'default') {
	443	+public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response) {
176	444	if (!current_user_can('manage_options')) {
	445	+ //error_log(esc_html__('Unauthorized PDF processing attempt', 'mxchat'));
177	446	return false;
178	447	}
179	448
180	449	$pdf_url = esc_url_raw($pdf_url);
		@@ -180,8 +449,9 @@
180	449	$pdf_url = esc_url_raw($pdf_url);
181	450	$upload_dir = wp_upload_dir();
182	451
183	452	if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
	453	+ //error_log(sprintf(esc_html__('Upload directory error: %s', 'mxchat'), esc_html($upload_dir['error'])));
184	454	return false;
185	455	}
186	456
187	457	$pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
		@@ -188,12 +458,14 @@
188	458	$pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
189	459
190	460	$response_body = wp_remote_retrieve_body($response);
191	461	if (empty($response_body)) {
	462	+ //error_log(esc_html__('Empty PDF response body', 'mxchat'));
192	463	return false;
193	464	}
194	465
195	466	if (!wp_mkdir_p(dirname($pdf_path))) {
	467	+ //error_log(sprintf(esc_html__('Failed to create directory for PDF: %s', 'mxchat'), esc_html($pdf_path)));
196	468	return false;
197	469	}
198	470
199	471	try {
		@@ -202,693 +474,295 @@
202	474	if (!file_exists($pdf_path)) {
203	475	throw new Exception(__('Failed to save PDF file', 'mxchat'));
204	476	}
205	477
206		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
207		-
208		- if ($total_pages === false || $total_pages < 1) {
209		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
210		- }
	478	+ $parser = new \Smalot\PdfParser\Parser();
	479	+ $pdf = $parser->parseFile($pdf_path);
	480	+ $total_pages = absint(count($pdf->getPages()));
211	481
212		- // Create unique queue ID
213		- $queue_id = 'pdf_' . md5($pdf_url . time());
214		-
215		- // Create array of pages to process
216		- $pages = array();
217		- for ($i = 1; $i <= $total_pages; $i++) {
218		- $pages[] = array(
219		- 'pdf_path' => $pdf_path,
220		- 'pdf_url' => $pdf_url,
221		- 'page_number' => $i,
222		- 'total_pages' => $total_pages
223		- );
	482	+ if ($total_pages < 1) {
	483	+ throw new Exception(__('Invalid PDF: no pages found', 'mxchat'));
224	484	}
225	485
226		- // Add pages to queue
227		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
	486	+ wp_schedule_single_event(time(), 'mxchat_process_pdf_pages', array(
	487	+ 'pdf_path' => $pdf_path,
	488	+ 'pdf_url' => $pdf_url,
	489	+ 'total_pages' => $total_pages,
	490	+ 'batch_size' => absint(15),
	491	+ 'batch_pause' => absint(10)
	492	+ ));
228	493
229		- if ($queued_count === 0) {
230		- wp_delete_file($pdf_path);
231		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
232		- }
	494	+ $status_data = array(
	495	+ 'total_pages' => $total_pages,
	496	+ 'processed_pages' => 0,
	497	+ 'status' => 'processing',
	498	+ 'last_update' => time()
	499	+ );
233	500
234		- // Store queue metadata
235		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $pdf_url);
236		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
237		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
238		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
239		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
240		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	501	+ set_transient(
	502	+ sanitize_key('mxchat_pdf_status_' . md5($pdf_url)),
	503	+ array_map('sanitize_text_field', $status_data),
	504	+ DAY_IN_SECONDS
	505	+ );
241	506
242		- // Store queue ID in transient for status tracking
243		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
244		- set_transient('mxchat_last_pdf_url', $pdf_url, DAY_IN_SECONDS);
	507	+ return __('scheduled', 'mxchat');
245	508
246		- return 'queued';
247		-
248	509	} catch (Exception $e) {
	510	+ //error_log(sprintf(esc_html__('Error preparing PDF for processing: %s', 'mxchat'), esc_html($e->getMessage())));
249	511	if (file_exists($pdf_path)) {
250	512	wp_delete_file($pdf_path);
251	513	}
252		- return $e->getMessage();
	514	+ return false;
253	515	}
254	516	}
255	517
256		-/**
257		- * Handle direct PDF file upload from the knowledge base page
258		- */
259		-public function mxchat_handle_pdf_file_submission() {
260		- if (!isset($_POST['submit_pdf_file']) || !current_user_can('manage_options')) {
261		- wp_die(esc_html__('Unauthorized access', 'mxchat'));
262		- }
	518	+public function mxchat_process_pdf_pages_cron($pdf_path, $pdf_url, $total_pages, $batch_size, $batch_pause) {
	519	+ // Validate inputs
	520	+ $pdf_path = sanitize_text_field($pdf_path);
	521	+ $pdf_url = esc_url_raw($pdf_url);
	522	+ $total_pages = absint($total_pages);
	523	+ $batch_size = absint($batch_size);
	524	+ $batch_pause = absint($batch_pause);
263	525
264		- check_admin_referer('mxchat_submit_pdf_file_action', 'mxchat_submit_pdf_file_nonce');
265		-
266		- $redirect_url = admin_url('admin.php?page=mxchat-prompts');
267		-
268		- // Validate file upload
269		- if (empty($_FILES['pdf_file']) || $_FILES['pdf_file']['error'] !== UPLOAD_ERR_OK) {
270		- $error_code = isset($_FILES['pdf_file']['error']) ? $_FILES['pdf_file']['error'] : UPLOAD_ERR_NO_FILE;
271		- $error_messages = array(
272		- UPLOAD_ERR_INI_SIZE => __('The uploaded file exceeds the server upload_max_filesize limit.', 'mxchat'),
273		- UPLOAD_ERR_FORM_SIZE => __('The uploaded file exceeds the form MAX_FILE_SIZE limit.', 'mxchat'),
274		- UPLOAD_ERR_PARTIAL => __('The file was only partially uploaded.', 'mxchat'),
275		- UPLOAD_ERR_NO_FILE => __('No file was uploaded. Please select a PDF file.', 'mxchat'),
276		- UPLOAD_ERR_NO_TMP_DIR => __('Server missing temporary folder.', 'mxchat'),
277		- UPLOAD_ERR_CANT_WRITE => __('Server failed to write file to disk.', 'mxchat'),
278		- );
279		- $error_msg = isset($error_messages[$error_code]) ? $error_messages[$error_code] : __('Unknown upload error.', 'mxchat');
280		- set_transient('mxchat_admin_notice_error', $error_msg, 30);
281		- wp_safe_redirect(esc_url($redirect_url));
282		- exit;
283		- }
284		-
285		- $file = $_FILES['pdf_file'];
286		-
287		- // Validate MIME type
288		- $finfo = finfo_open(FILEINFO_MIME_TYPE);
289		- $mime_type = finfo_file($finfo, $file['tmp_name']);
290		- finfo_close($finfo);
291		-
292		- if ($mime_type !== 'application/pdf') {
293		- set_transient('mxchat_admin_notice_error',
294		- esc_html__('Invalid file type. Only PDF files are accepted.', 'mxchat'),
295		- 30
296		- );
297		- wp_safe_redirect(esc_url($redirect_url));
298		- exit;
299		- }
300		-
301		- // Validate extension
302		- $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
303		- if ($ext !== 'pdf') {
304		- set_transient('mxchat_admin_notice_error',
305		- esc_html__('Invalid file extension. Only .pdf files are accepted.', 'mxchat'),
306		- 30
307		- );
308		- wp_safe_redirect(esc_url($redirect_url));
309		- exit;
310		- }
311		-
312		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
313		- $original_filename = sanitize_file_name($file['name']);
314		-
315		- $upload_dir = wp_upload_dir();
316		- if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
317		- set_transient('mxchat_admin_notice_error',
318		- esc_html__('WordPress upload directory is not writable.', 'mxchat'),
319		- 30
320		- );
321		- wp_safe_redirect(esc_url($redirect_url));
322		- exit;
323		- }
324		-
325		- $pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
326		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
327		-
328		- if (!wp_mkdir_p(dirname($pdf_path))) {
329		- set_transient('mxchat_admin_notice_error',
330		- esc_html__('Failed to create upload directory.', 'mxchat'),
331		- 30
332		- );
333		- wp_safe_redirect(esc_url($redirect_url));
334		- exit;
335		- }
336		-
337		- // Move uploaded file
338		- if (!move_uploaded_file($file['tmp_name'], $pdf_path)) {
339		- set_transient('mxchat_admin_notice_error',
340		- esc_html__('Failed to save uploaded PDF file.', 'mxchat'),
341		- 30
342		- );
343		- wp_safe_redirect(esc_url($redirect_url));
344		- exit;
345		- }
346		-
347	526	try {
348		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
349		-
350		- if ($total_pages === false || $total_pages < 1) {
351		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
	527	+ if (!file_exists($pdf_path)) {
	528	+ throw new Exception(sprintf('PDF file not found at path: %s', esc_html($pdf_path)));
352	529	}
353	530
354		- // Use original filename as the source identifier
355		- $source_label = 'upload://' . $original_filename;
356		-
357		- $queue_id = 'pdf_' . md5($source_label . time());
358		-
359		- $pages = array();
360		- for ($i = 1; $i <= $total_pages; $i++) {
361		- $pages[] = array(
362		- 'pdf_path' => $pdf_path,
363		- 'pdf_url' => $source_label,
364		- 'page_number' => $i,
365		- 'total_pages' => $total_pages,
366		- );
367		- }
368		-
369		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
370		-
371		- if ($queued_count === 0) {
372		- wp_delete_file($pdf_path);
373		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
374		- }
375		-
376		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $source_label);
377		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
378		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
379		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
380		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
381		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
382		-
383		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
384		- set_transient('mxchat_last_pdf_url', $source_label, DAY_IN_SECONDS);
385		-
386		- set_transient('mxchat_admin_notice_success',
387		- sprintf(
388		- esc_html__('PDF "%s" (%d pages) queued for processing. Processing will start automatically.', 'mxchat'),
389		- esc_html($original_filename),
390		- $total_pages
391		- ),
392		- 30
393		- );
394		-
395		- } catch (Exception $e) {
396		- if (file_exists($pdf_path)) {
397		- wp_delete_file($pdf_path);
398		- }
399		- set_transient('mxchat_admin_notice_error',
400		- esc_html__('Failed to process uploaded PDF: ', 'mxchat') . esc_html($e->getMessage()),
401		- 30
402		- );
403		- }
404		-
405		- wp_safe_redirect(esc_url($redirect_url));
406		- exit;
407		-}
408		-
409		-/**
410		- * Validate PDF and count pages with multiple parser attempts
411		- */
412		-private function mxchat_validate_and_count_pdf_pages($pdf_path) {
413		- // Method 1: Try with Smalot PDF Parser (your current method)
414		- try {
415		- mxchat_load_pdf_parser();
416	531	$parser = new \Smalot\PdfParser\Parser();
417	532	$pdf = $parser->parseFile($pdf_path);
418	533	$pages = $pdf->getPages();
419		- $page_count = count($pages);
420		-
421		- if ($page_count > 0) {
422		- //error_log('PDF parsed successfully with Smalot parser: ' . $page_count . ' pages');
423		- return $page_count;
424		- }
425		- } catch (Exception $e) {
426		- //error_log('Smalot PDF parser failed: ' . $e->getMessage());
427		- }
428	534
429		- // Method 2: Try with pdfinfo command (if available)
430		- if (function_exists('shell_exec') && !$this->mxchat_is_shell_disabled()) {
431		- try {
432		- $command = 'pdfinfo ' . escapeshellarg($pdf_path) . ' 2>&1';
433		- $output = shell_exec($command);
434		-
435		- if ($output && preg_match('/Pages:\s*(\d+)/', $output, $matches)) {
436		- $page_count = intval($matches[1]);
437		- if ($page_count > 0) {
438		- //error_log('PDF parsed successfully with pdfinfo: ' . $page_count . ' pages');
439		- return $page_count;
440		- }
441		- }
442		- } catch (Exception $e) {
443		- //error_log('pdfinfo command failed: ' . $e->getMessage());
	535	+ // Get current progress
	536	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	537	+ $status = get_transient($status_key);
	538	+
	539	+ if (!$status || !is_array($status)) {
	540	+ throw new Exception('Invalid status data retrieved from transient');
444	541	}
445		- }
446	542
447		- // Method 3: Try to repair PDF and parse again
448		- try {
449		- $repaired_path = $this->mxchat_attempt_pdf_repair($pdf_path);
450		- if ($repaired_path && $repaired_path !== $pdf_path) {
451		- mxchat_load_pdf_parser();
452		- $parser = new \Smalot\PdfParser\Parser();
453		- $pdf = $parser->parseFile($repaired_path);
454		- $pages = $pdf->getPages();
455		- $page_count = count($pages);
456		-
457		- if ($page_count > 0) {
458		- // Replace original with repaired version
459		- copy($repaired_path, $pdf_path);
460		- unlink($repaired_path);
461		- //error_log('PDF repaired and parsed successfully: ' . $page_count . ' pages');
462		- return $page_count;
463		- }
464		-
465		- // Clean up repaired file if it didn't work
466		- unlink($repaired_path);
	543	+ // Initialize failed pages list if it doesn't exist
	544	+ if (!isset($status['failed_pages_list']) || !is_array($status['failed_pages_list'])) {
	545	+ $status['failed_pages_list'] = [];
467	546	}
468		- } catch (Exception $e) {
469		- //error_log('PDF repair attempt failed: ' . $e->getMessage());
470		- }
471	547
472		- // Method 4: Manual PDF structure analysis (basic page count)
473		- try {
474		- $page_count = $this->mxchat_manual_pdf_page_count($pdf_path);
475		- if ($page_count > 0) {
476		- //error_log('PDF page count determined manually: ' . $page_count . ' pages');
477		- return $page_count;
	548	+ $start_page = absint($status['processed_pages']);
	549	+ $end_page = min($start_page + $batch_size, $total_pages);
	550	+ $options = get_option('mxchat_options');
	551	+
	552	+ if (empty($options['api_key'])) {
	553	+ throw new Exception('API key is missing or invalid');
478	554	}
479		- } catch (Exception $e) {
480		- //error_log('Manual PDF analysis failed: ' . $e->getMessage());
481		- }
482	555
483		- //error_log('All PDF parsing methods failed for: ' . $pdf_path);
484		- return false;
485		-}
	556	+ $successful_pages = 0;
	557	+ $failed_pages = 0;
486	558
487		-/**
488		- * Check if shell_exec is disabled
489		- */
490		-private function mxchat_is_shell_disabled() {
491		- $disabled = explode(',', ini_get('disable_functions'));
492		- return in_array('shell_exec', $disabled);
493		-}
	559	+ for ($i = $start_page; $i < $end_page; $i++) {
	560	+ $page_number = $i + 1;
	561	+ $max_retries = 3;
	562	+ $retry_count = 0;
	563	+ $page_processed = false;
	564	+ $last_error = '';
494	565
495		-/**
496		- * Attempt to repair PDF using basic methods
497		- */
498		-private function mxchat_attempt_pdf_repair($pdf_path) {
499		- try {
500		- $content = file_get_contents($pdf_path);
501		- if (!$content) {
502		- return false;
503		- }
	566	+ while (!$page_processed && $retry_count < $max_retries) {
	567	+ try {
	568	+ $text = $pages[$i]->getText();
	569	+
	570	+ if (empty($text)) {
	571	+ throw new Exception("Empty text on page {$page_number}");
	572	+ }
	573	+
	574	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
504	575
505		- // Check if PDF starts with proper header
506		- if (substr($content, 0, 4) !== '%PDF') {
507		- // Try to find PDF header in the content
508		- $header_pos = strpos($content, '%PDF');
509		- if ($header_pos !== false && $header_pos < 1024) {
510		- // Remove junk before PDF header
511		- $content = substr($content, $header_pos);
512		- $repaired_path = $pdf_path . '.repaired';
513		- file_put_contents($repaired_path, $content);
514		- return $repaired_path;
515		- }
516		- }
	576	+ if (empty($sanitized_content)) {
	577	+ throw new Exception("No valid content after sanitization on page {$page_number}");
	578	+ }
517	579
518		- // Check for EOF marker
519		- $content = rtrim($content);
520		- if (!preg_match('/%%EOF\s*$/', $content)) {
521		- // Add EOF marker if missing
522		- $content .= "\n%%EOF";
523		- $repaired_path = $pdf_path . '.repaired';
524		- file_put_contents($repaired_path, $content);
525		- return $repaired_path;
526		- }
	580	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	581	+
	582	+ if (is_string($embedding_vector)) {
	583	+ throw new Exception("Embedding generation failed: " . $embedding_vector);
	584	+ }
	585	+
	586	+ if (!is_array($embedding_vector)) {
	587	+ throw new Exception("Embedding generation returned unexpected result type: " . gettype($embedding_vector));
	588	+ }
	589	+
	590	+ $metadata = array(
	591	+ 'document_type' => 'pdf',
	592	+ 'total_pages' => $total_pages,
	593	+ 'current_page' => $page_number,
	594	+ 'prev_page' => $i > 0 ? $i : null,
	595	+ 'next_page' => $i < ($total_pages - 1) ? $i + 2 : null,
	596	+ 'source_url' => $pdf_url
	597	+ );
527	598
528		- } catch (Exception $e) {
529		- //error_log('PDF repair error: ' . $e->getMessage());
530		- }
	599	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	600	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
531	601
532		- return false;
533		-}
	602	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $options['api_key']);
	603	+
	604	+ if (is_wp_error($db_result)) {
	605	+ throw new Exception("Database submission failed: " . $db_result->get_error_message());
	606	+ }
534	607
535		-/**
536		- * Manual PDF page counting by analyzing PDF structure
537		- */
538		-private function mxchat_manual_pdf_page_count($pdf_path) {
539		- try {
540		- $content = file_get_contents($pdf_path);
541		- if (!$content) {
542		- return 0;
543		- }
	608	+ // Success!
	609	+ $page_processed = true;
	610	+ $successful_pages++;
	611	+
	612	+ } catch (Exception $e) {
	613	+ $retry_count++;
	614	+ $last_error = $e->getMessage();
	615	+
	616	+ //error_log("PDF page {$page_number} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	617	+
	618	+ if ($retry_count < $max_retries) {
	619	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	620	+ sleep(pow(2, $retry_count - 1));
	621	+ }
	622	+ }
	623	+ }
544	624
545		- // Method 1: Count /Type /Page objects
546		- $page_count = preg_match_all('/\/Type\s*\/Page[^s]/', $content);
547		- if ($page_count > 0) {
548		- return $page_count;
549		- }
	625	+ // If page still not processed after all retries, mark as failed
	626	+ if (!$page_processed) {
	627	+ $failed_pages++;
	628	+ $status['failed_pages_list'][] = [
	629	+ 'page' => $page_number,
	630	+ 'error' => $last_error,
	631	+ 'time' => time(),
	632	+ 'retries' => $max_retries
	633	+ ];
	634	+
	635	+ // Limit failed pages list to prevent memory issues
	636	+ if (count($status['failed_pages_list']) > 50) {
	637	+ $status['failed_pages_list'] = array_slice($status['failed_pages_list'], -50);
	638	+ }
	639	+
	640	+ //error_log("PDF page {$page_number} permanently failed after {$max_retries} attempts: " . $last_error);
	641	+ }
550	642
551		- // Method 2: Look for /Count in pages object
552		- if (preg_match('/\/Type\s*\/Pages.*?\/Count\s+(\d+)/', $content, $matches)) {
553		- return intval($matches[1]);
	643	+ // Update progress
	644	+ $status['processed_pages'] = absint($page_number);
	645	+ $status['last_update'] = time();
	646	+ $status['failed_pages'] = absint($status['failed_pages'] ?? 0) + ($page_processed ? 0 : 1);
	647	+
	648	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
554	649	}
555	650
556		- // Method 3: Count page references
557		- $page_count = preg_match_all('/\d+\s+0\s+obj\s*<<[^>]*\/Type\s*\/Page/', $content);
558		- if ($page_count > 0) {
559		- return $page_count;
560		- }
561		-
562		- } catch (Exception $e) {
563		- //error_log('Manual PDF analysis error: ' . $e->getMessage());
564		- }
565		-
566		- return 0;
567		-}
568		-
569		-
570		-public function mxchat_save_inline_prompt() {
571		- // DEBUG: Log what we're receiving
572		- //error_log('=== MXCHAT DEBUG ===');
573		- //error_log('POST data: ' . print_r($_POST, true));
574		- //error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
575		-
576		- // Check for nonce security
577		- check_ajax_referer('mxchat_save_inline_nonce', '_ajax_nonce');
578		-
579		- // If we get here, nonce passed
580		- //error_log('Nonce verification PASSED');
581		-
582		- // Verify permissions
583		- if (!current_user_can('manage_options')) {
584		- wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
585		- return;
586		- }
587		-
588		- global $wpdb;
589		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
590		-
591		- // Validate and sanitize input data
592		- $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
593		- $article_content = isset($_POST['article_content']) ? wp_kses_post(wp_unslash($_POST['article_content'])) : '';
594		- $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
595		-
596		- if ($prompt_id > 0 && !empty($article_content)) {
597		- // Re-generate the embedding vector for the updated content
598		- $embedding_vector = $this->mxchat_generate_embedding($article_content);
599		- if (is_array($embedding_vector)) {
600		- // Serialize the embedding vector before storing it
601		- $embedding_vector_serialized = serialize($embedding_vector);
602		- // Update the prompt in the database
603		- $updated = $wpdb->update(
604		- $table_name,
605		- array(
606		- 'article_content' => $article_content,
607		- 'embedding_vector' => $embedding_vector_serialized,
608		- 'source_url' => $article_url,
609		- ),
610		- array('id' => $prompt_id),
611		- array('%s', '%s', '%s'),
612		- array('%d')
613		- );
614		- if ($updated !== false) {
615		- wp_send_json_success();
616		- } else {
617		- MxChat_Admin::mxchat_log_debug('knowledge_error', 'Database update failed when saving knowledge entry');
618		- wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
619		- }
	651	+ // Schedule next batch if needed
	652	+ if ($end_page < $total_pages) {
	653	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_pdf_pages', array(
	654	+ 'pdf_path' => $pdf_path,
	655	+ 'pdf_url' => $pdf_url,
	656	+ 'total_pages' => $total_pages,
	657	+ 'batch_size' => $batch_size,
	658	+ 'batch_pause' => $batch_pause
	659	+ ));
620	660	} else {
621		- MxChat_Admin::mxchat_log_debug('embedding_error', 'Embedding generation failed for knowledge entry');
622		- wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
623		- }
624		- } else {
625		- wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
626		- }
627		-}
628		-
629		-
630		-/**
631		- * AJAX: Get full content for editing — reassembles chunks if needed.
632		- * Works for both WordPress DB and Pinecone entries.
633		- */
634		-public function ajax_mxchat_get_entry_content() {
635		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
636		-
637		- if ( ! current_user_can('manage_options') ) {
638		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
639		- }
640		-
641		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
642		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
643		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
644		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
645		-
646		- if ( $data_source === 'pinecone' ) {
647		- // Pinecone: fetch vectors by source_url, reassemble chunks
648		- $content = $this->get_pinecone_entry_content( $source_url, $entry_id, $bot_id );
649		- } else {
650		- // WordPress DB
651		- $content = $this->get_wordpress_entry_content( $source_url, $entry_id );
652		- }
653		-
654		- if ( is_wp_error( $content ) ) {
655		- wp_send_json_error( array( 'message' => $content->get_error_message() ) );
656		- }
657		-
658		- wp_send_json_success( $content );
659		-}
660		-
661		-/**
662		- * Get content from WordPress DB — reassembles chunks by source_url.
663		- */
664		-private function get_wordpress_entry_content( $source_url, $entry_id ) {
665		- global $wpdb;
666		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
667		-
668		- // If we have a source_url, check for chunks
669		- if ( ! empty( $source_url ) && strpos( $source_url, 'mxchat://' ) !== 0 ) {
670		- $rows = $wpdb->get_results( $wpdb->prepare(
671		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
672		- $source_url
673		- ) );
674		-
675		- if ( $rows && count( $rows ) > 1 ) {
676		- // Multiple rows = chunked. Reassemble.
677		- $chunks = array();
678		- foreach ( $rows as $row ) {
679		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
680		- $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
681		- $chunks[ $index ] = $parsed['text'];
	661	+ // Processing complete
	662	+ $status['status'] = 'complete';
	663	+ $status['processed_pages'] = $total_pages;
	664	+
	665	+ // Add completion summary
	666	+ $status['completion_summary'] = [
	667	+ 'total_pages' => $total_pages,
	668	+ 'successful_pages' => $total_pages - absint($status['failed_pages'] ?? 0),
	669	+ 'failed_pages' => absint($status['failed_pages'] ?? 0),
	670	+ 'completion_time' => current_time('mysql')
	671	+ ];
	672	+
	673	+ // Save the completed status (don't delete it - let user dismiss manually)
	674	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	675	+
	676	+ // Clean up the temporary PDF file
	677	+ if (file_exists($pdf_path)) {
	678	+ wp_delete_file($pdf_path);
682	679	}
683		- ksort( $chunks );
684		- return array(
685		- 'content' => implode( "\n\n", $chunks ),
686		- 'source_url' => $source_url,
687		- 'is_chunked' => true,
688		- 'chunk_count' => count( $chunks ),
689		- 'content_type' => $rows[0]->content_type,
690		- );
691		- } elseif ( $rows && count( $rows ) === 1 ) {
692		- $parsed = MxChat_Chunker::parse_stored_chunk( $rows[0]->article_content );
693		- return array(
694		- 'content' => $parsed['text'],
695		- 'source_url' => $source_url,
696		- 'entry_id' => $rows[0]->id,
697		- 'is_chunked' => false,
698		- 'content_type' => $rows[0]->content_type,
699		- );
	680	+
	681	+ // DON'T delete the status transients here - let user dismiss manually
700	682	}
701		- }
702	683
703		- // Fallback: fetch by ID
704		- if ( $entry_id > 0 ) {
705		- $row = $wpdb->get_row( $wpdb->prepare(
706		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
707		- $entry_id
708		- ) );
709		- if ( $row ) {
710		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
711		- return array(
712		- 'content' => $parsed['text'],
713		- 'source_url' => $row->source_url,
714		- 'entry_id' => $row->id,
715		- 'is_chunked' => false,
716		- 'content_type' => $row->content_type,
	684	+ } catch (\Exception $e) {
	685	+ //error_log(sprintf('[MXCHAT-PDF] Error processing PDF: %s', $e->getMessage()));
	686	+
	687	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	688	+ $status = get_transient($status_key);
	689	+
	690	+ if (!$status || !is_array($status)) {
	691	+ $status = array(
	692	+ 'total_pages' => $total_pages,
	693	+ 'processed_pages' => 0,
	694	+ 'status' => 'error',
	695	+ 'error' => sanitize_text_field($e->getMessage()),
	696	+ 'last_update' => time()
717	697	);
	698	+ } else {
	699	+ $status['status'] = 'error';
	700	+ $status['error'] = sanitize_text_field($e->getMessage());
	701	+ $status['last_update'] = time();
718	702	}
719		- }
720		-
721		- return new WP_Error( 'not_found', 'Entry not found.' );
722		-}
723		-
724		-/**
725		- * Get content from Pinecone — fetches vectors by source_url, reassembles chunks.
726		- */
727		-private function get_pinecone_entry_content( $source_url, $entry_id, $bot_id ) {
728		- if ( ! class_exists('MxChat_Pinecone_Manager') ) {
729		- return new WP_Error( 'pinecone_unavailable', 'Pinecone manager not available.' );
730		- }
731		-
732		- // Get Pinecone config
733		- if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
734		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
735		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
736		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
737		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
738		- } else {
739		- $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
740		- $api_key = $bot_config['api_key'] ?? '';
741		- $host = $bot_config['host'] ?? '';
742		- $namespace = $bot_config['namespace'] ?? '';
743		- }
744		-
745		- if ( empty($host) || empty($api_key) ) {
746		- return new WP_Error( 'pinecone_config', 'Pinecone not configured.' );
747		- }
748		-
749		- // List vectors with the source_url prefix
750		- $base_id = md5( $source_url );
751		- $vector_ids = array( $base_id );
752		-
753		- // Find chunk vectors
754		- $list_url = "https://{$host}/vectors/list";
755		- $list_body = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
756		- if ( ! empty($namespace) ) {
757		- $list_body['namespace'] = $namespace;
758		- }
759		-
760		- $list_resp = wp_remote_post( $list_url, array(
761		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
762		- 'body' => wp_json_encode( $list_body ),
763		- 'timeout' => 15,
764		- ) );
765		-
766		- if ( ! is_wp_error($list_resp) ) {
767		- $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
768		- if ( ! empty($list_data['vectors']) ) {
769		- foreach ( $list_data['vectors'] as $v ) {
770		- $vector_ids[] = $v['id'];
771		- }
	703	+
	704	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	705	+
	706	+ if (file_exists($pdf_path)) {
	707	+ wp_delete_file($pdf_path);
772	708	}
773	709	}
774		-
775		- // Fetch vectors with metadata
776		- $fetch_url = "https://{$host}/vectors/fetch";
777		- $fetch_body = array( 'ids' => $vector_ids );
778		- if ( ! empty($namespace) ) {
779		- $fetch_body['namespace'] = $namespace;
780		- }
781		-
782		- $fetch_resp = wp_remote_post( $fetch_url, array(
783		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
784		- 'body' => wp_json_encode( $fetch_body ),
785		- 'timeout' => 15,
786		- ) );
787		-
788		- if ( is_wp_error($fetch_resp) ) {
789		- return new WP_Error( 'pinecone_fetch', 'Failed to fetch from Pinecone.' );
790		- }
791		-
792		- $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
793		- $vectors = $fetch_data['vectors'] ?? array();
794		-
795		- if ( empty($vectors) ) {
796		- return new WP_Error( 'not_found', 'Entry not found in Pinecone.' );
797		- }
798		-
799		- // Reassemble chunks
800		- $chunks = array();
801		- $content_type = 'content';
802		- foreach ( $vectors as $vid => $vector ) {
803		- $meta = $vector['metadata'] ?? array();
804		- $text = $meta['text'] ?? '';
805		- $index = $meta['chunk_index'] ?? 0;
806		- $content_type = $meta['type'] ?? 'content';
807		- $chunks[ intval($index) ] = $text;
808		- }
809		- ksort( $chunks );
810		-
811		- return array(
812		- 'content' => implode( "\n\n", $chunks ),
813		- 'source_url' => $source_url,
814		- 'is_chunked' => count($chunks) > 1,
815		- 'chunk_count' => count($chunks),
816		- 'content_type' => $content_type,
817		- );
818	710	}
819	711
820		-/**
821		- * AJAX: Save edited content — re-chunks and re-embeds as needed.
822		- * Works for both WordPress DB and Pinecone entries.
823		- */
824		-public function ajax_mxchat_save_entry_content() {
825		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
	712	+ public function mxchat_save_inline_prompt() {
	713	+ // Check for nonce security
	714	+ check_ajax_referer('mxchat_save_inline_nonce');
826	715
827		- if ( ! current_user_can('manage_options') ) {
828		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
829		- }
	716	+ // Verify permissions
	717	+ if (!current_user_can('manage_options')) {
	718	+ wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
	719	+ return;
	720	+ }
830	721
831		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
832		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
833		- $content = isset($_POST['content']) ? wp_kses_post( wp_unslash($_POST['content']) ) : '';
834		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
835		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
836		- $content_type = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : 'content';
	722	+ global $wpdb;
	723	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
837	724
838		- if ( empty($content) ) {
839		- wp_send_json_error( array( 'message' => 'Content cannot be empty.' ) );
840		- }
	725	+ // Validate and sanitize input data
	726	+ $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
	727	+ $article_content = isset($_POST['article_content']) ? sanitize_textarea_field($_POST['article_content']) : '';
	728	+ $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
841	729
842		- // Get the embedding API key
843		- $options = get_option('mxchat_options', array());
844		- $api_key = '';
	730	+ if ($prompt_id > 0 && !empty($article_content)) {
	731	+ // Re-generate the embedding vector for the updated content
	732	+ $embedding_vector = $this->mxchat_generate_embedding($article_content);
845	733
846		- if ( $bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager') ) {
847		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
848		- $api_key = $bot_options['api_key'] ?? '';
849		- }
850		- if ( empty($api_key) ) {
851		- $api_key = $options['api_key'] ?? '';
852		- }
	734	+ if (is_array($embedding_vector)) {
	735	+ // Serialize the embedding vector before storing it
	736	+ $embedding_vector_serialized = serialize($embedding_vector);
853	737
854		- global $wpdb;
855		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
	738	+ // Update the prompt in the database
	739	+ $updated = $wpdb->update(
	740	+ $table_name,
	741	+ array(
	742	+ 'article_content' => $article_content,
	743	+ 'embedding_vector' => $embedding_vector_serialized,
	744	+ 'source_url' => $article_url,
	745	+ ),
	746	+ array('id' => $prompt_id),
	747	+ array('%s', '%s', '%s'),
	748	+ array('%d')
	749	+ );
856	750
857		- // If source_url is empty but we have an entry_id, look it up
858		- if ( empty($source_url) && $entry_id > 0 && $data_source === 'wordpress' ) {
859		- $row = $wpdb->get_row( $wpdb->prepare( "SELECT source_url FROM {$table} WHERE id = %d", $entry_id ) );
860		- if ( $row && ! empty($row->source_url) ) {
861		- $source_url = $row->source_url;
862		- }
863		- }
	751	+ if ($updated !== false) {
	752	+ wp_send_json_success();
	753	+ } else {
	754	+ wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
	755	+ }
	756	+ } else {
	757	+ wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
	758	+ }
	759	+ } else {
	760	+ wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
	761	+ }
	762	+ }
864	763
865		- // For manual entries (no source_url or mxchat:// prefix), delete the old entry by ID first
866		- // so submit_content_to_db creates a replacement instead of a duplicate
867		- // Also treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
868		- $is_legacy_manual = !empty($source_url) && strpos($source_url, 'mxchat.ai') !== false && strpos($source_url, 'mxchat://') !== 0;
869		- if ( $entry_id > 0 && (empty($source_url) || strpos($source_url, 'mxchat://') === 0 || $is_legacy_manual) ) {
870		- $wpdb->delete( $table, array( 'id' => $entry_id ), array( '%d' ) );
871		- // Clear legacy URL so submit_content_to_db generates a unique mxchat:// identifier
872		- // instead of reusing the shared URL (which would mass-delete other entries with the same URL)
873		- if ( $is_legacy_manual ) {
874		- $source_url = '';
875		- }
876		- }
877	764
878		- // Use the existing submit_content_to_db which handles chunking, Pinecone, and WP DB
879		- $vector_id = ! empty($source_url) ? md5($source_url) : md5('mxchat_manual_' . $entry_id);
880		-
881		- // submit_content_to_db already handles: delete old chunks → re-chunk → re-embed → store
882		- $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, $vector_id, $bot_id, $content_type );
883		-
884		- if ( is_wp_error($result) ) {
885		- wp_send_json_error( array( 'message' => $result->get_error_message() ) );
886		- }
887		-
888		- wp_send_json_success( array( 'message' => 'Content saved and re-embedded successfully.' ) );
889		-}
890		-
891	765	public function mxchat_get_pdf_processing_status($pdf_url) {
892	766	$pdf_url = esc_url_raw($pdf_url);
893	767	$status = get_transient(sanitize_key('mxchat_pdf_status_' . md5($pdf_url)));
894	768
		@@ -931,18 +805,24 @@
931	805	}
932	806
933	807
934	808	public function mxchat_handle_sitemap_submission() {
	809	+ // Start logging the submission process
	810	+ //error_log('[MXCHAT-URL] ===== Starting URL submission process =====');
	811	+
935	812	// Check if the form was submitted and verify permissions
936	813	if (!isset($_POST['submit_sitemap']) || !current_user_can('manage_options')) {
	814	+ //error_log('[MXCHAT-URL] Error: Unauthorized access or form not submitted properly');
937	815	wp_die(esc_html__('Unauthorized access', 'mxchat'));
938	816	}
939	817
940	818	// Verify nonce
	819	+ //error_log('[MXCHAT-URL] Verifying nonce');
941	820	check_admin_referer('mxchat_submit_sitemap_action', 'mxchat_submit_sitemap_nonce');
942	821
943	822	// Validate URL
944	823	if (!isset($_POST['sitemap_url']) || empty($_POST['sitemap_url'])) {
	824	+ //error_log('[MXCHAT-URL] Error: Empty or missing URL');
945	825	set_transient('mxchat_admin_notice_error',
946	826	esc_html__('Please provide a valid URL.', 'mxchat'),
947	827	30
948	828	);
		@@ -950,28 +830,12 @@
950	830	exit;
951	831	}
952	832
953	833	$submitted_url = esc_url_raw($_POST['sitemap_url']);
	834	+ //error_log('[MXCHAT-URL] Processing URL: ' . $submitted_url);
954	835
955		- // Convert Google Drive sharing URLs to direct download URLs
956		- if ( strpos($submitted_url, 'drive.google.com') !== false ) {
957		- $file_id = '';
958		- if ( preg_match('/[?&]id=([a-zA-Z0-9_-]+)/', $submitted_url, $m) ) {
959		- $file_id = $m[1];
960		- } elseif ( preg_match('#/file/d/([a-zA-Z0-9_-]+)#', $submitted_url, $m) ) {
961		- $file_id = $m[1];
962		- }
963		- if ( ! empty($file_id) ) {
964		- $submitted_url = 'https://drive.google.com/uc?export=download&id=' . $file_id;
965		- }
966		- }
967		-
968		- // Get bot_id from form submission
969		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
970		-
971		- // Get bot-specific options and validate API key
972		- $bot_options = $this->get_bot_options($bot_id);
973		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	836	+ // Validate API key first
	837	+ $options = get_option('mxchat_options');
974	838	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
975	839
976	840	if (strpos($selected_model, 'voyage') === 0) {
977	841	$api_key = $options['voyage_api_key'] ?? '';
		@@ -988,26 +852,21 @@
988	852	$error_message = sprintf(
989	853	esc_html__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
990	854	$provider_name
991	855	);
	856	+ //error_log('[MXCHAT-URL] Error: ' . $error_message);
992	857	set_transient('mxchat_admin_notice_error', $error_message, 30);
	858	+ //error_log('[MXCHAT-URL] Set error transient: ' . $error_message);
993	859	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
994	860	exit;
995	861	}
996	862
997		- // Fetch URL — use browser-like headers so servers with bot protection don't block us
998		- $response = wp_remote_get($submitted_url, array(
999		- 'timeout' => 30,
1000		- 'sslverify' => false,
1001		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
1002		- 'headers' => array(
1003		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1004		- 'Accept-Language' => 'en-US,en;q=0.9',
1005		- ),
1006		- ));
	863	+ //error_log('[MXCHAT-URL] Fetching URL content');
	864	+ $response = wp_remote_get($submitted_url, array('timeout' => 30));
1007	865
1008	866	if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
1009	867	$error_message = is_wp_error($response) ? $response->get_error_message() : 'HTTP Status: ' . wp_remote_retrieve_response_code($response);
	868	+ //error_log('[MXCHAT-URL] Error fetching URL: ' . $error_message);
1010	869	set_transient('mxchat_admin_notice_error',
1011	870	sprintf(
1012	871	esc_html__('Failed to fetch the URL: %s', 'mxchat'),
1013	872	esc_html($error_message)
		@@ -1018,11 +877,13 @@
1018	877	exit;
1019	878	}
1020	879
1021	880	$content_type = wp_remote_retrieve_header($response, 'content-type');
	881	+ //error_log('[MXCHAT-URL] Content type: ' . $content_type);
1022	882	$body_content = wp_remote_retrieve_body($response);
1023	883
1024	884	if (empty($body_content)) {
	885	+ //error_log('[MXCHAT-URL] Error: Empty response body');
1025	886	set_transient('mxchat_admin_notice_error',
1026	887	esc_html__('Empty response received from URL.', 'mxchat'),
1027	888	30
1028	889	);
		@@ -1028,21 +889,29 @@
1028	889	);
1029	890	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1030	891	exit;
1031	892	}
	893	+ //error_log('[MXCHAT-URL] Retrieved body content length: ' . strlen($body_content) . ' bytes');
1032	894
1033	895	// Handle PDF URL
1034	896	if ($this->mxchat_is_pdf_url($submitted_url, $response)) {
1035		- $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response, $bot_id);
	897	+ //error_log('[MXCHAT-URL] Detected PDF URL, handling PDF for knowledge base');
	898	+ $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response);
	899	+ //error_log('[MXCHAT-URL] PDF handling result: ' . $result);
1036	900
1037		- if ($result === 'queued') {
1038		- set_transient('mxchat_admin_notice_success',
1039		- esc_html__('PDF queued for processing. Processing will start automatically.', 'mxchat'),
	901	+ if ($result === 'scheduled') {
	902	+ set_transient(
	903	+ 'mxchat_last_pdf_url',
	904	+ sanitize_text_field($submitted_url),
	905	+ DAY_IN_SECONDS
	906	+ );
	907	+ set_transient('mxchat_admin_notice_info',
	908	+ esc_html__('PDF processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1040	909	30
1041	910	);
1042	911	} else {
1043	912	set_transient('mxchat_admin_notice_error',
1044		- esc_html__('Failed to queue PDF processing: ', 'mxchat') . esc_html($result),
	913	+ esc_html__('Failed to start PDF processing: ', 'mxchat') . esc_html($result),
1045	914	30
1046	915	);
1047	916	}
1048	917
		@@ -1051,8 +920,9 @@
1051	920	}
1052	921
1053	922	// Handle Sitemap XML
1054	923	if (strpos($content_type, 'xml') !== false || strpos($body_content, '<urlset') !== false) {
	924	+ //error_log('[MXCHAT-URL] Detected XML content, processing as sitemap');
1055	925	libxml_use_internal_errors(true);
1056	926	$xml = simplexml_load_string($body_content);
1057	927	$xml_errors = libxml_get_errors();
1058	928	libxml_clear_errors();
		@@ -1057,8 +927,15 @@
1057	927	$xml_errors = libxml_get_errors();
1058	928	libxml_clear_errors();
1059	929
1060	930	if ($xml === false || !empty($xml_errors)) {
	931	+ //error_log('[MXCHAT-URL] Error: Invalid XML format');
	932	+ if (!empty($xml_errors)) {
	933	+ foreach ($xml_errors as $error) {
	934	+ //error_log('[MXCHAT-URL] XML Error: ' . $error->message);
	935	+ }
	936	+ }
	937	+
1061	938	set_transient('mxchat_admin_notice_error',
1062	939	esc_html__('Invalid sitemap XML. Please provide a valid sitemap.', 'mxchat'),
1063	940	30
1064	941	);
		@@ -1065,18 +942,27 @@
1065	942	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1066	943	exit;
1067	944	}
1068	945
1069		- $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url, $bot_id);
	946	+ //error_log('[MXCHAT-URL] Valid XML found, handling sitemap for knowledge base');
	947	+ $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url);
	948	+ //error_log('[MXCHAT-URL] Sitemap handling result: ' . $result);
1070	949
1071		- if ($result === 'queued') {
1072		- set_transient('mxchat_admin_notice_success',
1073		- esc_html__('Sitemap queued for processing. Processing will start automatically.', 'mxchat'),
	950	+ if ($result === 'scheduled') {
	951	+ set_transient(
	952	+ 'mxchat_last_sitemap_url',
	953	+ sanitize_text_field($submitted_url),
	954	+ DAY_IN_SECONDS
	955	+ );
	956	+ set_transient('mxchat_admin_notice_info',
	957	+ esc_html__('Sitemap processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1074	958	30
1075	959	);
1076	960	} else {
	961	+ // Return to the admin page without a redirect for better error display
	962	+ // The error is already stored in the sitemap status transient
1077	963	set_transient('mxchat_admin_notice_error',
1078		- esc_html__('Failed to queue sitemap processing. Please check the status below for details.', 'mxchat'),
	964	+ esc_html__('Failed to start sitemap processing. Please check the status below for details.', 'mxchat'),
1079	965	30
1080	966	);
1081	967	}
1082	968
		@@ -1083,50 +969,127 @@
1083	969	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1084	970	exit;
1085	971	}
1086	972
1087		- // Handle Regular URL (single page)
	973	+ // Handle Regular URL
	974	+ //error_log('[MXCHAT-URL] Processing as regular webpage');
1088	975	$page_content = $this->mxchat_extract_main_content($body_content);
	976	+ //error_log('[MXCHAT-URL] Extracted content length: ' . strlen($page_content) . ' bytes');
	977	+
1089	978	$sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	979	+ //error_log('[MXCHAT-URL] Sanitized content length: ' . strlen($sanitized_content) . ' bytes');
1090	980
1091		- //error_log('[MXCHAT-URL-DEBUG] URL: ' . $submitted_url);
1092		- //error_log('[MXCHAT-URL-DEBUG] Extracted page_content length: ' . strlen($page_content));
1093		- //error_log('[MXCHAT-URL-DEBUG] Sanitized content length: ' . strlen($sanitized_content));
1094		- //error_log('[MXCHAT-URL-DEBUG] Content preview: ' . substr($sanitized_content, 0, 500));
1095		-
1096	981	if (empty($sanitized_content)) {
	982	+ //error_log('[MXCHAT-URL] Error: No valid content after sanitization');
	983	+
	984	+ // Set both transients - the error notice and the URL status
1097	985	set_transient('mxchat_admin_notice_error',
1098	986	esc_html__('No valid content found on the provided URL.', 'mxchat'),
1099	987	30
1100	988	);
	989	+
	990	+ // Set URL status transient
	991	+ set_transient('mxchat_single_url_status', [
	992	+ 'url' => $submitted_url,
	993	+ 'timestamp' => current_time('mysql'),
	994	+ 'status' => 'failed',
	995	+ 'error' => esc_html__('No valid content found on the provided URL.', 'mxchat')
	996	+ ], DAY_IN_SECONDS);
	997	+
1101	998	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1102	999	exit;
1103	1000	}
1104	1001
1105		- // For single URLs, process immediately using submit_content_to_db
1106		- // This handles chunking automatically for large content
1107		- $db_result = MxChat_Utils::submit_content_to_db(
1108		- $sanitized_content,
1109		- $submitted_url,
1110		- $api_key,
1111		- null,
1112		- $bot_id,
1113		- 'url' // content_type
1114		- );
	1002	+ //error_log('[MXCHAT-URL] Generating embedding for content');
	1003	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
1115	1004
1116		- if (is_wp_error($db_result)) {
1117		- $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1005	+ // Check if embedding_vector is a string (error message)
	1006	+ if (is_string($embedding_vector)) {
	1007	+ //error_log('[MXCHAT-URL] Error generating embedding: ' . $embedding_vector);
	1008	+ $error_message = esc_html__('Failed to generate embedding: ', 'mxchat') . esc_html($embedding_vector);
	1009	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1010	+
	1011	+ // Set both transients
1118	1012	set_transient('mxchat_admin_notice_error', $error_message, 30);
1119		- } else {
	1013	+
	1014	+ // Set URL status transient
	1015	+ set_transient('mxchat_single_url_status', [
	1016	+ 'url' => $submitted_url,
	1017	+ 'timestamp' => current_time('mysql'),
	1018	+ 'status' => 'failed',
	1019	+ 'error' => $error_message
	1020	+ ], DAY_IN_SECONDS);
	1021	+
	1022	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1023	+ exit;
	1024	+ }
	1025	+
	1026	+ if (is_array($embedding_vector)) {
	1027	+ //error_log('[MXCHAT-URL] Successfully generated embedding with ' . count($embedding_vector) . ' dimensions');
	1028	+
	1029	+ $db_result = MxChat_Utils::submit_content_to_db(
	1030	+ $sanitized_content,
	1031	+ $submitted_url,
	1032	+ $api_key
	1033	+ );
	1034	+
	1035	+ if (is_wp_error($db_result)) {
	1036	+ //error_log('[MXCHAT-URL] Error: Failed to store content in database: ' . $db_result->get_error_message());
	1037	+ $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1038	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1039	+
	1040	+ // Set both transients
	1041	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1042	+
	1043	+ // Set URL status transient
	1044	+ set_transient('mxchat_single_url_status', [
	1045	+ 'url' => $submitted_url,
	1046	+ 'timestamp' => current_time('mysql'),
	1047	+ 'status' => 'failed',
	1048	+ 'error' => $error_message
	1049	+ ], DAY_IN_SECONDS);
	1050	+
	1051	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1052	+ exit;
	1053	+ }
	1054	+
	1055	+ //error_log('[MXCHAT-URL] Successfully stored content in database');
1120	1056	$success_message = esc_html__('URL content successfully submitted!', 'mxchat');
	1057	+ //error_log('[MXCHAT-URL] Setting success transient: ' . $success_message);
	1058	+
	1059	+ // Set both transients
1121	1060	set_transient('mxchat_admin_notice_success', $success_message, 30);
	1061	+
	1062	+ // Set URL status transient with success
	1063	+ set_transient('mxchat_single_url_status', [
	1064	+ 'url' => $submitted_url,
	1065	+ 'timestamp' => current_time('mysql'),
	1066	+ 'status' => 'complete',
	1067	+ 'content_length' => strlen($sanitized_content),
	1068	+ 'embedding_dimensions' => count($embedding_vector)
	1069	+ ], DAY_IN_SECONDS);
	1070	+
	1071	+ } else {
	1072	+ //error_log('[MXCHAT-URL] Error: Failed to generate embedding. Unexpected result type: ' . gettype($embedding_vector));
	1073	+ $error_message = esc_html__('Failed to generate embedding: Unexpected result type. Please check your API key and try again.', 'mxchat');
	1074	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1075	+
	1076	+ // Set both transients
	1077	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1078	+
	1079	+ // Set URL status transient
	1080	+ set_transient('mxchat_single_url_status', [
	1081	+ 'url' => $submitted_url,
	1082	+ 'timestamp' => current_time('mysql'),
	1083	+ 'status' => 'failed',
	1084	+ 'error' => $error_message
	1085	+ ], DAY_IN_SECONDS);
1122	1086	}
1123	1087
	1088	+ //error_log('[MXCHAT-URL] ===== Completed URL submission process =====');
1124	1089	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1125	1090	exit;
1126	1091	}
1127		-
1128		-
1129	1092	public function mxchat_get_single_url_status() {
1130	1093	$status = get_transient('mxchat_single_url_status');
1131	1094	if (!$status) {
1132	1095	return null;
		@@ -1138,11 +1101,13 @@
1138	1101	}
1139	1102
1140	1103	return $status;
1141	1104	}
1142		-
1143		-public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url, $bot_id = 'default') {
	1105	+public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url) {
	1106	+ // Clear any single URL status when starting sitemap processing
	1107	+ delete_transient('mxchat_single_url_status');
1144	1108	if (!current_user_can('manage_options')) {
	1109	+ //error_log(esc_html__('Unauthorized sitemap processing attempt', 'mxchat'));
1145	1110	return false;
1146	1111	}
1147	1112
1148	1113	try {
		@@ -1151,96 +1116,331 @@
1151	1116	if (!$xml || !is_object($xml)) {
1152	1117	throw new Exception(__('Invalid XML object provided', 'mxchat'));
1153	1118	}
1154	1119
1155		- // Get bot-specific embedding API for validation
1156		- $bot_options = $this->get_bot_options($bot_id);
1157		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	1120	+ // Add embedding validation before processing
	1121	+ // Test embedding with a small sample text to verify API key is working
	1122	+ $test_result = $this->mxchat_generate_embedding("This is a test to verify the embedding API key is working.");
1158	1123
1159		- // Test the embedding API before processing
1160		- $test_phrase = "Test embedding generation for MxChat";
1161		- $test_result = $this->mxchat_generate_embedding($test_phrase, $bot_id);
1162		-
	1124	+ // Check if test_result is a string (error message) rather than an array (valid embedding)
1163	1125	if (is_string($test_result)) {
	1126	+ //error_log('[MXCHAT-URL] Embedding API validation failed: ' . $test_result);
	1127	+
	1128	+ // Store the error in the status transient so it can be displayed later
	1129	+ $status_data = array(
	1130	+ 'total_urls' => 0,
	1131	+ 'processed_urls' => 0,
	1132	+ 'status' => 'error',
	1133	+ 'error' => __('Embedding API validation failed: ', 'mxchat') . $test_result,
	1134	+ 'last_update' => time()
	1135	+ );
	1136	+
	1137	+ set_transient(
	1138	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1139	+ array_map('sanitize_text_field', $status_data),
	1140	+ DAY_IN_SECONDS
	1141	+ );
	1142	+
1164	1143	throw new Exception(__('Embedding API validation failed: ', 'mxchat') . $test_result);
1165	1144	}
1166	1145
	1146	+ // Make sure it's an array (valid embedding)
1167	1147	if (!is_array($test_result)) {
	1148	+ //error_log('[MXCHAT-URL] Embedding API returned unexpected result type: ' . gettype($test_result));
1168	1149	throw new Exception(__('Embedding API returned unexpected result type. Please check your configuration.', 'mxchat'));
1169	1150	}
1170	1151
1171		- // Extract URLs from sitemap
1172		- $urls = array();
	1152	+ $urls = [];
1173	1153	foreach ($xml->url as $url_element) {
1174	1154	$url = esc_url_raw((string)$url_element->loc);
1175	1155	if ($url) {
1176		- $urls[] = array('url' => $url);
	1156	+ $urls[] = $url;
1177	1157	}
1178	1158	}
1179	1159
1180		- $total_urls = count($urls);
	1160	+ $total_urls = absint(count($urls));
1181	1161
1182	1162	if ($total_urls < 1) {
1183	1163	throw new Exception(__('No valid URLs found in sitemap', 'mxchat'));
1184	1164	}
1185	1165
1186		- // Create unique queue ID
1187		- $queue_id = 'sitemap_' . md5($sitemap_url . time());
	1166	+ wp_schedule_single_event(time(), 'mxchat_process_sitemap_urls', array(
	1167	+ 'urls' => $urls,
	1168	+ 'sitemap_url' => $sitemap_url,
	1169	+ 'total_urls' => $total_urls,
	1170	+ 'batch_size' => absint(10),
	1171	+ 'batch_pause' => absint(5)
	1172	+ ));
1188	1173
1189		- // Add URLs to queue
1190		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'url', $urls, $bot_id);
	1174	+ $status_data = array(
	1175	+ 'total_urls' => $total_urls,
	1176	+ 'processed_urls' => 0,
	1177	+ 'status' => 'processing',
	1178	+ 'last_update' => time()
	1179	+ );
1191	1180
1192		- if ($queued_count === 0) {
1193		- throw new Exception(__('Failed to add URLs to processing queue', 'mxchat'));
1194		- }
	1181	+ set_transient(
	1182	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1183	+ array_map('sanitize_text_field', $status_data),
	1184	+ DAY_IN_SECONDS
	1185	+ );
1195	1186
1196		- // Store queue metadata
1197		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $sitemap_url);
1198		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'sitemap');
1199		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_urls);
1200		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
1201		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	1187	+ return __('scheduled', 'mxchat');
1202	1188
1203		- // Store queue ID in transient for status tracking
1204		- set_transient('mxchat_active_queue_sitemap', $queue_id, DAY_IN_SECONDS);
1205		- set_transient('mxchat_last_sitemap_url', $sitemap_url, DAY_IN_SECONDS);
1206		-
1207		- return 'queued';
1208		-
1209		- } catch (Exception $e) {
	1189	+ } catch (\Exception $e) {
1210	1190	$error_message = $e->getMessage();
1211	1191	//error_log(sprintf(esc_html__('Error preparing sitemap for processing: %s', 'mxchat'), esc_html($error_message)));
1212	1192
	1193	+ // Store the sitemap URL and error in transients so they can be displayed
	1194	+ set_transient(
	1195	+ 'mxchat_last_sitemap_url',
	1196	+ sanitize_text_field($sitemap_url),
	1197	+ DAY_IN_SECONDS
	1198	+ );
	1199	+
	1200	+ $status_data = array(
	1201	+ 'total_urls' => 0,
	1202	+ 'processed_urls' => 0,
	1203	+ 'status' => 'error',
	1204	+ 'error' => $error_message,
	1205	+ 'last_update' => time()
	1206	+ );
	1207	+
	1208	+ set_transient(
	1209	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1210	+ array_map('sanitize_text_field', $status_data),
	1211	+ DAY_IN_SECONDS
	1212	+ );
	1213	+
1213	1214	return $error_message;
1214	1215	}
1215		-
1216	1216	}
1217	1217
1218		-/**
1219		- * Remove shortcode tags but preserve the content inside them
1220		- * Example: [vc_column]Hello World[/vc_column] becomes "Hello World"
1221		- *
1222		- * @param string $content The content containing shortcodes
1223		- * @return string Content with shortcode tags removed but inner content preserved
1224		- */
1225		-private function strip_shortcode_tags_preserve_content($content) {
1226		- // Single-pass regex removes all shortcode brackets: [tag], [tag attr="val"], [/tag], [tag /]
1227		- // Content between tags is inherently preserved since only brackets are targeted
1228		- $result = preg_replace('/\[\/?\w[\w-]*[^\]]*\]/', '', $content);
1229		- return ($result !== null) ? $result : $content;
	1218	+public function mxchat_process_sitemap_urls_cron($urls, $sitemap_url, $total_urls, $batch_size, $batch_pause) {
	1219	+ // Validate inputs
	1220	+ $sitemap_url = esc_url_raw($sitemap_url);
	1221	+ $total_urls = absint($total_urls);
	1222	+ $batch_size = absint($batch_size);
	1223	+ $batch_pause = absint($batch_pause);
	1224	+
	1225	+ if (!is_array($urls) || empty($urls)) {
	1226	+ return;
	1227	+ }
	1228	+
	1229	+ try {
	1230	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	1231	+ $status = get_transient($status_key);
	1232	+
	1233	+ if (!$status || !is_array($status)) {
	1234	+ throw new Exception('Invalid status data retrieved from transient');
	1235	+ }
	1236	+
	1237	+ // Initialize failed_urls array if it doesn't exist
	1238	+ if (!isset($status['failed_urls_list']) || !is_array($status['failed_urls_list'])) {
	1239	+ $status['failed_urls_list'] = [];
	1240	+ }
	1241	+
	1242	+ $start_url = absint($status['processed_urls']);
	1243	+ $end_url = min($start_url + $batch_size, $total_urls);
	1244	+
	1245	+ // Track batch statistics
	1246	+ $batch_stats = [
	1247	+ 'processed' => 0,
	1248	+ 'failed' => 0,
	1249	+ 'last_error' => '',
	1250	+ 'embedding_errors' => 0
	1251	+ ];
	1252	+
	1253	+ // Check embedding configuration with first URL (only on first batch)
	1254	+ if ($start_url === 0) {
	1255	+ $test_url = esc_url_raw($urls[0]);
	1256	+ $test_response = wp_remote_get($test_url);
	1257	+
	1258	+ if (!is_wp_error($test_response) && wp_remote_retrieve_response_code($test_response) === 200) {
	1259	+ $test_html = wp_remote_retrieve_body($test_response);
	1260	+ $test_content = $this->mxchat_extract_main_content($test_html);
	1261	+ $test_sanitized = $this->mxchat_sanitize_content_for_api($test_content);
	1262	+
	1263	+ if (!empty($test_sanitized)) {
	1264	+ $test_embedding = $this->mxchat_generate_embedding($test_sanitized);
	1265	+
	1266	+ if (is_string($test_embedding)) {
	1267	+ throw new Exception('Embedding generation failed: ' . $test_embedding);
	1268	+ }
	1269	+
	1270	+ if (!is_array($test_embedding)) {
	1271	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($test_embedding));
	1272	+ }
	1273	+ }
	1274	+ }
	1275	+ }
	1276	+
	1277	+ for ($i = $start_url; $i < $end_url; $i++) {
	1278	+ $page_url = esc_url_raw($urls[$i]);
	1279	+ $max_retries = 3;
	1280	+ $retry_count = 0;
	1281	+ $url_processed = false;
	1282	+ $last_error = '';
	1283	+
	1284	+ while (!$url_processed && $retry_count < $max_retries) {
	1285	+ try {
	1286	+ // Attempt to fetch the URL
	1287	+ $page_response = wp_remote_get($page_url, array('timeout' => 30));
	1288	+
	1289	+ if (is_wp_error($page_response)) {
	1290	+ throw new Exception('HTTP request failed: ' . $page_response->get_error_message());
	1291	+ }
	1292	+
	1293	+ $response_code = wp_remote_retrieve_response_code($page_response);
	1294	+ if ($response_code !== 200) {
	1295	+ throw new Exception('HTTP Status: ' . $response_code);
	1296	+ }
	1297	+
	1298	+ $page_html = wp_remote_retrieve_body($page_response);
	1299	+
	1300	+ if (empty($page_html)) {
	1301	+ throw new Exception('Empty response body');
	1302	+ }
	1303	+
	1304	+ $page_content = $this->mxchat_extract_main_content($page_html);
	1305	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	1306	+
	1307	+ if (empty($sanitized_content)) {
	1308	+ throw new Exception('No valid content found after processing');
	1309	+ }
	1310	+
	1311	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	1312	+
	1313	+ if (is_string($embedding_vector)) {
	1314	+ throw new Exception('Embedding generation failed: ' . $embedding_vector);
	1315	+ }
	1316	+
	1317	+ if (!is_array($embedding_vector)) {
	1318	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($embedding_vector));
	1319	+ }
	1320	+
	1321	+ // Submit to database
	1322	+ $options = get_option('mxchat_options');
	1323	+ $submission_result = MxChat_Utils::submit_content_to_db($sanitized_content, $page_url, $options['api_key']);
	1324	+
	1325	+ if (is_wp_error($submission_result)) {
	1326	+ throw new Exception('Database submission failed: ' . $submission_result->get_error_message());
	1327	+ }
	1328	+
	1329	+ // Success!
	1330	+ $url_processed = true;
	1331	+ $batch_stats['processed']++;
	1332	+
	1333	+ } catch (Exception $e) {
	1334	+ $retry_count++;
	1335	+ $last_error = $e->getMessage();
	1336	+
	1337	+ //error_log("URL {$page_url} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	1338	+
	1339	+ // Track embedding errors specifically
	1340	+ if (strpos($last_error, 'Embedding') !== false) {
	1341	+ $batch_stats['embedding_errors']++;
	1342	+ }
	1343	+
	1344	+ if ($retry_count < $max_retries) {
	1345	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	1346	+ sleep(pow(2, $retry_count - 1));
	1347	+ }
	1348	+ }
	1349	+ }
	1350	+
	1351	+ // If URL still not processed after all retries, mark as failed
	1352	+ if (!$url_processed) {
	1353	+ $batch_stats['failed']++;
	1354	+ $batch_stats['last_error'] = $last_error;
	1355	+
	1356	+ // Add to failed URLs list
	1357	+ $status['failed_urls_list'][] = [
	1358	+ 'url' => $page_url,
	1359	+ 'error' => $last_error,
	1360	+ 'time' => time(),
	1361	+ 'retries' => $max_retries
	1362	+ ];
	1363	+
	1364	+ // Limit the number of failed URLs we store to prevent transient size issues
	1365	+ if (count($status['failed_urls_list']) > 100) {
	1366	+ $status['failed_urls_list'] = array_slice($status['failed_urls_list'], -100);
	1367	+ }
	1368	+ }
	1369	+
	1370	+ // Update progress
	1371	+ $status['processed_urls'] = absint($i + 1);
	1372	+ $status['last_update'] = time();
	1373	+ $status['failed_urls'] = absint($status['failed_urls'] ?? 0) + ($url_processed ? 0 : 1);
	1374	+ $status['last_error'] = $batch_stats['last_error'];
	1375	+
	1376	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1377	+
	1378	+ // If we have too many consecutive embedding errors, stop processing
	1379	+ if ($batch_stats['embedding_errors'] >= 10) {
	1380	+ throw new Exception('Too many consecutive embedding failures detected. Please check your API configuration.');
	1381	+ }
	1382	+ }
	1383	+
	1384	+ // If all URLs in this batch failed, stop processing
	1385	+ if ($batch_stats['processed'] === 0 && $batch_stats['failed'] > 0) {
	1386	+ $status['status'] = 'error';
	1387	+ $status['error'] = sprintf(
	1388	+ 'Processing stopped: %d consecutive failures in batch. Last error: %s',
	1389	+ $batch_stats['failed'],
	1390	+ $batch_stats['last_error']
	1391	+ );
	1392	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1393	+ return;
	1394	+ }
	1395	+
	1396	+ // Update final progress
	1397	+ $status['processed_urls'] = min($end_url, $total_urls);
	1398	+ $status['last_update'] = time();
	1399	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1400	+
	1401	+ // Check if we've processed all URLs
	1402	+ if ($end_url >= $total_urls) {
	1403	+ // All URLs have been processed - mark as complete
	1404	+ $status['status'] = 'complete';
	1405	+ $status['processed_urls'] = $total_urls;
	1406	+
	1407	+ // Add completion summary
	1408	+ $status['completion_summary'] = [
	1409	+ 'total_urls' => $total_urls,
	1410	+ 'successful_urls' => $total_urls - absint($status['failed_urls'] ?? 0),
	1411	+ 'failed_urls' => absint($status['failed_urls'] ?? 0),
	1412	+ 'completion_time' => current_time('mysql')
	1413	+ ];
	1414	+
	1415	+ // Save the completed status (don't delete it - let user dismiss manually)
	1416	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1417	+
	1418	+ // DON'T delete the status transients here - let user dismiss manually
	1419	+ } else {
	1420	+ // Schedule next batch
	1421	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_sitemap_urls', array(
	1422	+ 'urls' => $urls,
	1423	+ 'sitemap_url' => $sitemap_url,
	1424	+ 'total_urls' => $total_urls,
	1425	+ 'batch_size' => $batch_size,
	1426	+ 'batch_pause' => $batch_pause,
	1427	+ ));
	1428	+ }
	1429	+ } catch (\Exception $e) {
	1430	+ $status['status'] = 'error';
	1431	+ $status['error'] = $e->getMessage();
	1432	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1433	+ }
1230	1434	}
1231		-
1232	1435	public function mxchat_sanitize_content_for_api($content) {
1233	1436	//error_log('[MXCHAT-SANITIZE] Original content preview: ' . substr($content, 0, 500) . '...');
1234		-
1235		- // Remove shortcode tags but PRESERVE content inside them
1236		- $content = $this->strip_shortcode_tags_preserve_content($content);
1237		-
	1437	+
1238	1438	// Remove script, style tags, and HTML comments
1239	1439	$content = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $content);
1240	1440	$content = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $content);
1241	1441	$content = preg_replace('/<!--(.|\s)*?-->/', '', $content);
1242		-
	1442	+
1243	1443	// Remove all HTML tags and decode HTML entities
1244	1444	$content = wp_strip_all_tags($content);
1245	1445	$content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5);
1246	1446
		@@ -1298,12 +1498,12 @@
1298	1498	@$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'), LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
1299	1499	$xpath = new DOMXPath($dom);
1300	1500
1301	1501	// For debugging purposes
1302		- $debugEnabled = true; // Set to true to enable debugging output
	1502	+ $debugEnabled = false; // Set to true to enable debugging output
1303	1503	$debug = function($message) use ($debugEnabled) {
1304	1504	if ($debugEnabled) {
1305		- //error_log('[MXCHAT-EXTRACT-DEBUG] ' . $message);
	1505	+ //error_log('[MXCHAT-DEBUG] ' . $message);
1306	1506	}
1307	1507	};
1308	1508
1309	1509	// Direct targeting for Gerow theme posts
		@@ -1402,25 +1602,18 @@
1402	1602	'//main',
1403	1603	'//div[contains(@class, "content")]'
1404	1604	];
1405	1605
1406		- // First handle Elementor content - get only leaf widget containers to avoid duplicates
	1606	+ // First handle Elementor content
1407	1607	$debug("Checking for Elementor content");
1408		- // Get widget containers that are direct children of widgets (not nested inside other widget containers)
1409		- $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-widget")]//div[contains(@class, "elementor-widget-container")]');
	1608	+ $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-element")]//div[contains(@class, "elementor-widget-container")]');
1410	1609	if ($elementor_widgets && $elementor_widgets->length > 0) {
1411	1610	$debug("Found Elementor widgets");
1412		- $seen_content = array(); // Track seen content to avoid duplicates
1413	1611	$combined_content = '';
1414	1612	foreach ($elementor_widgets as $widget) {
1415	1613	$widget_content = $dom->saveHTML($widget);
1416	1614	if (!empty($widget_content)) {
1417		- // Create a hash of the content to detect duplicates
1418		- $content_hash = md5($widget_content);
1419		- if (!isset($seen_content[$content_hash])) {
1420		- $seen_content[$content_hash] = true;
1421		- $combined_content .= $widget_content;
1422		- }
	1615	+ $combined_content .= $widget_content;
1423	1616	}
1424	1617	}
1425	1618	if (!empty($combined_content)) {
1426	1619	$debug("Returning Elementor content");
		@@ -1432,14 +1625,15 @@
1432	1625	foreach ($selectors as $selector) {
1433	1626	$debug("Trying selector: " . $selector);
1434	1627	$nodes = $xpath->query($selector);
1435	1628	if ($nodes && $nodes->length > 0) {
1436		- $debug("Found " . $nodes->length . " matches for selector: " . $selector);
1437		- // Only take the FIRST matching node to avoid duplicate content
1438		- // (pages often have nested or multiple containers with same class)
1439		- $content = $dom->saveHTML($nodes->item(0));
	1629	+ $debug("Found matches for selector: " . $selector);
	1630	+ $content = '';
	1631	+ foreach ($nodes as $node) {
	1632	+ $content .= $dom->saveHTML($node);
	1633	+ }
1440	1634	if (!empty($content)) {
1441		- $debug("Returning content from selector: " . $selector . " (first match only)");
	1635	+ $debug("Returning content from selector: " . $selector);
1442	1636	return $content;
1443	1637	}
1444	1638	}
1445	1639	}
		@@ -1463,106 +1657,16 @@
1463	1657	$debug("Returning blog-area section content");
1464	1658	return $content;
1465	1659	}
1466	1660	}
1467		-
1468		- // Generic container selectors for non-CMS sites (like .asp pages)
1469		- $debug("Trying generic container selectors");
1470		- $generic_selectors = [
1471		- '//div[@id="main"]',
1472		- '//div[@id="wrapper"]',
1473		- '//div[@id="page"]',
1474		- '//div[@id="site-content"]',
1475		- '//div[contains(@class, "main-content")]',
1476		- '//div[contains(@class, "page-content")]',
1477		- '//div[contains(@class, "site-content")]',
1478		- ];
1479		-
1480		- foreach ($generic_selectors as $selector) {
1481		- $debug("Trying generic selector: " . $selector);
1482		- $nodes = $xpath->query($selector);
1483		- if ($nodes && $nodes->length > 0) {
1484		- $content = $dom->saveHTML($nodes->item(0));
1485		- if (!empty($content)) {
1486		- $debug("Returning content from generic selector: " . $selector);
1487		- return $content;
1488		- }
1489		- }
1490		- }
1491		-
1492		- // Paragraph-based content detection - find regions with substantial text
1493		- $debug("Trying paragraph-based content detection");
1494		- $paragraphs = $xpath->query('//p[string-length(normalize-space()) > 50]');
1495		- if ($paragraphs && $paragraphs->length >= 3) {
1496		- $debug("Found " . $paragraphs->length . " substantial paragraphs");
1497		- // Collect all substantial paragraphs and their content
1498		- $paragraph_content = '';
1499		- foreach ($paragraphs as $p) {
1500		- $paragraph_content .= $dom->saveHTML($p) . "\n";
1501		- }
1502		- if (!empty($paragraph_content)) {
1503		- $debug("Returning paragraph-based content");
1504		- return $paragraph_content;
1505		- }
1506		- }
1507		-
1508		- // Improved body fallback - strip nav/header/footer elements first
1509		- $debug("Using improved body fallback");
	1661	+
	1662	+ // Fallback: Return the body content if no specific selector matches
	1663	+ $debug("Using body fallback");
1510	1664	$body = $dom->getElementsByTagName('body');
1511	1665	if ($body->length > 0) {
1512		- // Clone the body to avoid modifying the original DOM
1513		- $body_clone = $body->item(0)->cloneNode(true);
1514		-
1515		- // Remove common non-content elements by tag name
1516		- $remove_tags = ['nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript'];
1517		- foreach ($remove_tags as $tag) {
1518		- $elements = $body_clone->getElementsByTagName($tag);
1519		- // Iterate backwards to safely remove elements
1520		- for ($i = $elements->length - 1; $i >= 0; $i--) {
1521		- $el = $elements->item($i);
1522		- if ($el && $el->parentNode) {
1523		- $el->parentNode->removeChild($el);
1524		- }
1525		- }
1526		- }
1527		-
1528		- // Remove elements with common non-content class names using XPath on the cloned body
1529		- $temp_dom = new DOMDocument();
1530		- @$temp_dom->appendChild($temp_dom->importNode($body_clone, true));
1531		- $temp_xpath = new DOMXPath($temp_dom);
1532		-
1533		- $remove_class_patterns = [
1534		- '//*[contains(@class, "nav")]',
1535		- '//*[contains(@class, "menu")]',
1536		- '//*[contains(@class, "sidebar")]',
1537		- '//*[contains(@class, "footer")]',
1538		- '//*[contains(@class, "header")]',
1539		- '//*[contains(@id, "nav")]',
1540		- '//*[contains(@id, "menu")]',
1541		- '//*[contains(@id, "sidebar")]',
1542		- '//*[contains(@id, "footer")]',
1543		- '//*[contains(@id, "header")]',
1544		- ];
1545		-
1546		- foreach ($remove_class_patterns as $pattern) {
1547		- $elements = $temp_xpath->query($pattern);
1548		- if ($elements) {
1549		- for ($i = $elements->length - 1; $i >= 0; $i--) {
1550		- $el = $elements->item($i);
1551		- if ($el && $el->parentNode) {
1552		- $el->parentNode->removeChild($el);
1553		- }
1554		- }
1555		- }
1556		- }
1557		-
1558		- $cleaned_content = $temp_dom->saveHTML();
1559		- if (!empty($cleaned_content)) {
1560		- $debug("Returning cleaned body content");
1561		- return $cleaned_content;
1562		- }
	1666	+ return $dom->saveHTML($body->item(0));
1563	1667	}
1564		-
	1668	+
1565	1669	// Last resort: return the original HTML
1566	1670	$debug("Returning original HTML");
1567	1671	return $html;
1568	1672	} catch (Exception $e) {
		@@ -1614,41 +1718,47 @@
1614	1718	try {
1615	1719	// Verify the request
1616	1720	check_ajax_referer('mxchat_status_nonce', 'nonce');
1617	1721
1618		- // Get active queue IDs
1619		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
1620		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
	1722	+ // Get the status just like in your admin page
	1723	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1724	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
1621	1725
1622		- $sitemap_status = false;
1623		- $pdf_status = false;
	1726	+ $pdf_status = $pdf_url ? $this->mxchat_get_pdf_processing_status($pdf_url) : false;
	1727	+ $sitemap_status = $sitemap_url ? $this->mxchat_get_sitemap_processing_status($sitemap_url) : false;
1624	1728
1625		- // Get sitemap queue status
1626		- if ($sitemap_queue_id) {
1627		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
	1729	+ // Add the PDF URL to the status object
	1730	+ if ($pdf_status && $pdf_url) {
	1731	+ $pdf_status['pdf_url'] = $pdf_url;
1628	1732	}
1629	1733
1630		- // Get PDF queue status
1631		- if ($pdf_queue_id) {
1632		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
1633		- }
	1734	+ // Set the current PDF URL for the manual batch processing button
	1735	+ $current_pdf_url = $pdf_url;
1634	1736
	1737	+ // Check for true processing status, not just presence of status
1635	1738	$is_active_processing =
1636		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
1637		- ($pdf_status && $pdf_status['status'] === 'processing');
	1739	+ ($sitemap_status && isset($sitemap_status['status']) && $sitemap_status['status'] === 'processing') ||
	1740	+ ($pdf_status && isset($pdf_status['status']) && $pdf_status['status'] === 'processing');
1638	1741
	1742	+ // Get single URL status, but only if no processing is active
	1743	+ $single_url_status = !$is_active_processing ? $this->mxchat_get_single_url_status() : false;
	1744	+
	1745	+ // REMOVED: Auto-clearing of completed status - now only done via dismiss button
	1746	+
1639	1747	// Return JSON response with the status data
1640	1748	wp_send_json(array(
1641	1749	'pdf_status' => $pdf_status,
1642	1750	'sitemap_status' => $sitemap_status,
	1751	+ 'single_url_status' => $single_url_status,
1643	1752	'is_processing' => $is_active_processing,
1644		- 'sitemap_queue_id' => $sitemap_queue_id,
1645		- 'pdf_queue_id' => $pdf_queue_id
	1753	+ 'current_pdf_url' => $current_pdf_url
1646	1754	));
1647	1755
1648	1756	} catch (Exception $e) {
	1757	+ // Log the error
1649	1758	//error_log('MxChat Status Update Error: ' . $e->getMessage());
1650	1759
	1760	+ // Return a friendly error response
1651	1761	wp_send_json_error(array(
1652	1762	'message' => 'Error getting status updates: ' . $e->getMessage(),
1653	1763	'status' => 'error'
1654	1764	));
		@@ -1653,1373 +1763,8 @@
1653	1763	'status' => 'error'
1654	1764	));
1655	1765	}
1656	1766	}
1657		-
1658		-/**
1659		- * Helper function to get queue status data
1660		- */
1661		-private function mxchat_get_queue_status_data($queue_id, $type = 'sitemap') {
1662		- global $wpdb;
1663		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
1664		-
1665		- // Get counts by status
1666		- $counts = $wpdb->get_results($wpdb->prepare(
1667		- "SELECT status, COUNT(*) as count
1668		- FROM $table_name
1669		- WHERE queue_id = %s
1670		- GROUP BY status",
1671		- $queue_id
1672		- ), OBJECT_K);
1673		-
1674		- $total = 0;
1675		- $completed = 0;
1676		- $failed = 0;
1677		- $processing = 0;
1678		- $pending = 0;
1679		-
1680		- foreach ($counts as $status => $data) {
1681		- $count = absint($data->count);
1682		- $total += $count;
1683		-
1684		- switch ($status) {
1685		- case 'completed':
1686		- $completed = $count;
1687		- break;
1688		- case 'failed':
1689		- $failed = $count;
1690		- break;
1691		- case 'processing':
1692		- $processing = $count;
1693		- break;
1694		- case 'pending':
1695		- $pending = $count;
1696		- break;
1697		- }
1698		- }
1699		-
1700		- if ($total === 0) {
1701		- return false;
1702		- }
1703		-
1704		- // Calculate percentage
1705		- $percentage = round((($completed + $failed) / $total) * 100);
1706		-
1707		- // Get failed items details (limit to 50)
1708		- $failed_items = array();
1709		- if ($failed > 0) {
1710		- $failed_results = $wpdb->get_results($wpdb->prepare(
1711		- "SELECT item_type, item_data, error_message, attempts, completed_at
1712		- FROM $table_name
1713		- WHERE queue_id = %s
1714		- AND status = 'failed'
1715		- AND attempts >= max_attempts
1716		- ORDER BY id DESC
1717		- LIMIT 50",
1718		- $queue_id
1719		- ));
1720		-
1721		- foreach ($failed_results as $item) {
1722		- $data = json_decode($item->item_data, true);
1723		- $url = $type === 'pdf' ? ($data['pdf_url'] ?? '') : ($data['url'] ?? '');
1724		-
1725		- $failed_items[] = array(
1726		- 'url' => $url,
1727		- 'page' => $type === 'pdf' ? ($data['page_number'] ?? 0) : 0,
1728		- 'error' => $item->error_message,
1729		- 'retries' => $item->attempts,
1730		- 'time' => strtotime($item->completed_at)
1731		- );
1732		- }
1733		- }
1734		-
1735		- // Get queue metadata
1736		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
1737		-
1738		- // Determine if queue is complete
1739		- $is_complete = ($pending === 0 && $processing === 0);
1740		-
1741		- // Get last update time
1742		- $last_update = $wpdb->get_var($wpdb->prepare(
1743		- "SELECT MAX(UNIX_TIMESTAMP(COALESCE(completed_at, started_at, created_at)))
1744		- FROM $table_name
1745		- WHERE queue_id = %s",
1746		- $queue_id
1747		- ));
1748		-
1749		- $last_update_text = $last_update ? human_time_diff($last_update, time()) . ' ' . __('ago', 'mxchat') : __('Just now', 'mxchat');
1750		-
1751		- // Format based on type
1752		- if ($type === 'pdf') {
1753		- return array(
1754		- 'total_pages' => $total,
1755		- 'processed_pages' => $completed + $failed,
1756		- 'failed_pages' => $failed,
1757		- 'percentage' => $percentage,
1758		- 'status' => $is_complete ? 'complete' : 'processing',
1759		- 'last_update' => $last_update_text,
1760		- 'failed_pages_list' => $failed_items,
1761		- 'pdf_url' => $source_url,
1762		- 'queue_id' => $queue_id
1763		- );
1764		- } else {
1765		- return array(
1766		- 'total_urls' => $total,
1767		- 'processed_urls' => $completed + $failed,
1768		- 'failed_urls' => $failed,
1769		- 'percentage' => $percentage,
1770		- 'status' => $is_complete ? 'complete' : 'processing',
1771		- 'last_update' => $last_update_text,
1772		- 'failed_urls_list' => $failed_items,
1773		- 'sitemap_url' => $source_url,
1774		- 'queue_id' => $queue_id
1775		- );
1776		- }
1777		-}
1778		-
1779		-/**
1780		- * Public method to get processing status for both sitemap and PDF queues
1781		- * Used by admin pages to display processing status
1782		- *
1783		- * @return array Array with 'sitemap_status', 'pdf_status', and 'is_processing' keys
1784		- */
1785		-public function mxchat_get_processing_statuses() {
1786		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
1787		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
1788		-
1789		- $sitemap_status = false;
1790		- $pdf_status = false;
1791		-
1792		- if ($sitemap_queue_id) {
1793		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
1794		- }
1795		-
1796		- if ($pdf_queue_id) {
1797		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
1798		- }
1799		-
1800		- $is_processing =
1801		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
1802		- ($pdf_status && $pdf_status['status'] === 'processing');
1803		-
1804		- return array(
1805		- 'sitemap_status' => $sitemap_status,
1806		- 'pdf_status' => $pdf_status,
1807		- 'is_processing' => $is_processing
1808		- );
1809		-}
1810		-
1811		-/**
1812		- * AJAX handler to get recent knowledge entries for real-time table updates
1813		- * UPDATED: Now supports both WordPress DB and Pinecone data sources
1814		- */
1815		-public function ajax_mxchat_get_recent_entries() {
1816		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
1817		-
1818		- if (!current_user_can('manage_options')) {
1819		- wp_send_json_error(array('message' => 'Unauthorized'));
1820		- return;
1821		- }
1822		-
1823		- global $wpdb;
1824		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1825		-
1826		- // Get parameters
1827		- $last_id = isset($_POST['last_id']) ? absint($_POST['last_id']) : 0;
1828		- $limit = isset($_POST['limit']) ? min(absint($_POST['limit']), 50) : 10;
1829		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
1830		-
1831		- // Check if Pinecone is enabled for this bot
1832		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
1833		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
1834		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
1835		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
1836		-
1837		- if ($use_pinecone && $has_pinecone_api) {
1838		- // PINECONE DATA SOURCE - Get grouped entry count (not raw vector count)
1839		- // Use mxchat_fetch_pinecone_records which returns total_unique_entries
1840		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, '', 1, 10, $bot_id, '');
1841		- $total_count = $records['total'] ?? 0;
1842		-
1843		- // For Pinecone, we don't return individual entries during polling
1844		- // (entries are already displayed on page load via mxchat_fetch_pinecone_records)
1845		- // We just return the updated count
1846		- wp_send_json_success(array(
1847		- 'entries' => array(),
1848		- 'total_count' => absint($total_count),
1849		- 'max_id' => $last_id,
1850		- 'data_source' => 'pinecone'
1851		- ));
1852		- return;
1853		- }
1854		-
1855		- // WORDPRESS DB DATA SOURCE
1856		- // Build query to get entries newer than last_id
1857		- $where_clauses = array('1=1');
1858		- $where_values = array();
1859		-
1860		- if ($last_id > 0) {
1861		- $where_clauses[] = 'id > %d';
1862		- $where_values[] = $last_id;
1863		- }
1864		-
1865		- // Note: WordPress DB table doesn't have bot_id column
1866		- // Multi-bot filtering is handled via Pinecone namespaces
1867		-
1868		- $where_sql = implode(' AND ', $where_clauses);
1869		-
1870		- // Get recent entries
1871		- $query = "SELECT id, article_content, source_url, timestamp
1872		- FROM $table_name
1873		- WHERE $where_sql
1874		- ORDER BY id DESC
1875		- LIMIT %d";
1876		-
1877		- $where_values[] = $limit;
1878		-
1879		- $entries = $wpdb->get_results($wpdb->prepare($query, $where_values));
1880		-
1881		- // Get total count of GROUPED entries (by source_url) - matches pagination display
1882		- // Count unique source_urls (excluding mxchat:// internal refs) + count of ungrouped rows
1883		- $total_count = $wpdb->get_var(
1884		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
1885		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
1886		- );
1887		-
1888		- // Format entries for response
1889		- $formatted_entries = array();
1890		- $preview_length = 150;
1891		- foreach ($entries as $entry) {
1892		- // Parse chunk metadata using the proper chunker method (same as initial page load)
1893		- if (class_exists('MxChat_Chunker')) {
1894		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($entry->article_content);
1895		- $display_content = $chunk_meta['text'];
1896		- $chunk_metadata = $chunk_meta['metadata'];
1897		- } else {
1898		- $display_content = $entry->article_content;
1899		- $chunk_metadata = array();
1900		- }
1901		-
1902		- $content_preview = mb_strlen($display_content) > $preview_length
1903		- ? mb_substr($display_content, 0, $preview_length) . '...'
1904		- : $display_content;
1905		-
1906		- $formatted_entries[] = array(
1907		- 'id' => $entry->id,
1908		- 'preview' => esc_html($content_preview),
1909		- 'full_content' => wp_kses_post(wpautop($display_content)),
1910		- 'content_length' => mb_strlen($display_content),
1911		- 'preview_length' => $preview_length,
1912		- 'source_url' => $entry->source_url,
1913		- 'has_link' => !empty($entry->source_url) && strpos($entry->source_url, 'mxchat://') !== 0,
1914		- 'chunk_metadata' => $chunk_metadata,
1915		- 'bot_id' => $entry->bot_id ?? 'default',
1916		- 'edit_nonce' => wp_create_nonce('mxchat_edit_entry_nonce'),
1917		- 'delete_nonce' => wp_create_nonce('mxchat_delete_prompt_nonce')
1918		- );
1919		- }
1920		-
1921		- wp_send_json_success(array(
1922		- 'entries' => $formatted_entries,
1923		- 'total_count' => absint($total_count),
1924		- 'max_id' => !empty($entries) ? $entries[0]->id : $last_id,
1925		- 'data_source' => 'wordpress'
1926		- ));
1927		-}
1928		-
1929		-/**
1930		- * Get Pinecone total count from stats API
1931		- * Helper function for ajax_mxchat_get_recent_entries
1932		- */
1933		-private function mxchat_get_pinecone_count_from_stats($pinecone_options) {
1934		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1935		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1936		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1937		-
1938		- if (empty($api_key) || empty($host)) {
1939		- return 0;
1940		- }
1941		-
1942		- try {
1943		- $stats_url = "https://{$host}/describe_index_stats";
1944		-
1945		- $response = wp_remote_post($stats_url, array(
1946		- 'headers' => array(
1947		- 'Api-Key' => $api_key,
1948		- 'Content-Type' => 'application/json'
1949		- ),
1950		- 'body' => '{}',
1951		- 'timeout' => 10
1952		- ));
1953		-
1954		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
1955		- $body = wp_remote_retrieve_body($response);
1956		- $stats_data = json_decode($body, true);
1957		-
1958		- // If namespace is specified, get count from that specific namespace
1959		- if (!empty($namespace) && isset($stats_data['namespaces'][$namespace]['vectorCount'])) {
1960		- return intval($stats_data['namespaces'][$namespace]['vectorCount']);
1961		- }
1962		-
1963		- // If no namespace specified or namespace not found in response, use total
1964		- return intval($stats_data['totalVectorCount'] ?? 0);
1965		- }
1966		-
1967		- return 0;
1968		-
1969		- } catch (Exception $e) {
1970		- return 0;
1971		- }
1972		-}
1973		-
1974		-/**
1975		- * AJAX handler to refresh Pinecone entries table via AJAX
1976		- * Returns the table HTML for updating the UI without a full page reload
1977		- */
1978		-public function ajax_mxchat_refresh_pinecone_entries() {
1979		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
1980		-
1981		- if (!current_user_can('manage_options')) {
1982		- wp_send_json_error(array('message' => 'Unauthorized'));
1983		- return;
1984		- }
1985		-
1986		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
1987		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
1988		- $per_page = 25;
1989		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
1990		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
1991		-
1992		- // Get Pinecone manager and options
1993		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
1994		- if (!$pinecone_manager) {
1995		- wp_send_json_error(array('message' => 'Pinecone manager not available'));
1996		- return;
1997		- }
1998		-
1999		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
2000		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2001		- $pinecone_api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2002		-
2003		- if (!$use_pinecone || empty($pinecone_api_key)) {
2004		- wp_send_json_error(array('message' => 'Pinecone not configured'));
2005		- return;
2006		- }
2007		-
2008		- // Fetch records from Pinecone
2009		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, $search_query, $page, $per_page, $bot_id, $content_type_filter);
2010		- $prompts = $records['data'] ?? array();
2011		- $total_records = $records['total'] ?? 0;
2012		-
2013		- // Preprocess Pinecone records — set chunk_metadata and display_content
2014		- // (matches admin-knowledge-page.php preprocessing)
2015		- foreach ($prompts as $prompt) {
2016		- if (isset($prompt->chunk_index) && $prompt->chunk_index !== null) {
2017		- $prompt->chunk_metadata = array(
2018		- 'chunk_index' => intval($prompt->chunk_index),
2019		- 'total_chunks' => isset($prompt->total_chunks) ? intval($prompt->total_chunks) : null,
2020		- 'is_chunked' => isset($prompt->is_chunked) ? (bool) $prompt->is_chunked : true,
2021		- 'source_url' => $prompt->source_url ?? ''
2022		- );
2023		- $prompt->display_content = $prompt->article_content;
2024		- } else {
2025		- $prompt->chunk_metadata = array();
2026		- $prompt->display_content = $prompt->article_content ?? '';
2027		- }
2028		- }
2029		-
2030		- // Group prompts by source_url
2031		- $grouped_prompts = array();
2032		- foreach ($prompts as $prompt) {
2033		- $source_url = '';
2034		- if (!empty($prompt->chunk_metadata['source_url'])) {
2035		- $source_url = $prompt->chunk_metadata['source_url'];
2036		- } elseif (!empty($prompt->source_url)) {
2037		- $source_url = $prompt->source_url;
2038		- }
2039		-
2040		- if (!empty($source_url)) {
2041		- if (!isset($grouped_prompts[$source_url])) {
2042		- $grouped_prompts[$source_url] = array();
2043		- }
2044		- $grouped_prompts[$source_url][] = $prompt;
2045		- } else {
2046		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2047		- }
2048		- }
2049		-
2050		- // Sort each group by chunk_index
2051		- foreach ($grouped_prompts as $source_url => &$group) {
2052		- usort($group, function($a, $b) {
2053		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2054		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2055		- return $index_a - $index_b;
2056		- });
2057		- }
2058		- unset($group);
2059		-
2060		- // Build HTML for the table rows - must match admin-knowledge-page.php structure exactly
2061		- ob_start();
2062		- $display_index = 0;
2063		- $current_page = $page;
2064		- $data_source = 'pinecone';
2065		- $current_bot_id = $bot_id;
2066		- $preview_length = 150;
2067		-
2068		- if (empty($grouped_prompts)) {
2069		- echo '<tr><td colspan="4" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2070		- esc_html_e('No knowledge entries found in Pinecone.', 'mxchat');
2071		- echo '</td></tr>';
2072		- } else {
2073		- foreach ($grouped_prompts as $source_url => $group) {
2074		- $chunk_count = count($group);
2075		- $first_prompt = $group[0];
2076		- $display_index++;
2077		-
2078		- if ($chunk_count > 1) {
2079		- // Multiple chunks - show grouped row with expand button
2080		- $group_id = 'group-' . md5($source_url);
2081		- ?>
2082		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2083		- class="mxchat-chunk-group-header"
2084		- data-source="<?php echo esc_attr($data_source); ?>"
2085		- data-group-id="<?php echo esc_attr($group_id); ?>"
2086		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2087		- <td style="padding: 12px 16px; text-align: center;">
2088		- <input type="checkbox"
2089		- class="mxchat-entry-checkbox"
2090		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2091		- data-source="<?php echo esc_attr($data_source); ?>"
2092		- data-source-url="<?php echo esc_attr($source_url); ?>"
2093		- data-is-group="true"
2094		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2095		- </td>
2096		- <td style="padding: 12px 16px; font-size: 13px;">
2097		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2098		- </td>
2099		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2100		- <div class="mxchat-chunk-group-info">
2101		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2102		- <span class="dashicons dashicons-arrow-right-alt2"></span>
2103		- </button>
2104		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2105		- <span class="mxchat-chunk-preview">
2106		- <?php
2107		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : (isset($first_prompt->article_content) ? $first_prompt->article_content : '');
2108		- $content_preview = mb_substr($parent_content, 0, 100);
2109		- echo esc_html($content_preview . '...');
2110		- ?>
2111		- </span>
2112		- </div>
2113		- </td>
2114		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2115		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2116		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2117		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2118		- <?php esc_html_e('View Source', 'mxchat'); ?>
2119		- </a>
2120		- <?php else : ?>
2121		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2122		- <?php endif; ?>
2123		- </td>
2124		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2125		- <?php if ($data_source !== 'pinecone') : ?>
2126		- <button type="button"
2127		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2128		- data-source-url="<?php echo esc_attr($source_url); ?>"
2129		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2130		- data-data-source="<?php echo esc_attr($data_source); ?>"
2131		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2132		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2133		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2134		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2135		- </button>
2136		- <?php endif; ?>
2137		- <button type="button"
2138		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2139		- data-source-url="<?php echo esc_attr($source_url); ?>"
2140		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2141		- data-data-source="<?php echo esc_attr($data_source); ?>"
2142		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2143		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2144		- style="color: var(--mxch-error);"
2145		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2146		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2147		- </button>
2148		- </td>
2149		- </tr>
2150		- <?php
2151		- // Render hidden chunk rows
2152		- foreach ($group as $chunk_index => $chunk) {
2153		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
2154		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
2155		- $content = isset($chunk->display_content) ? $chunk->display_content : (isset($chunk->article_content) ? $chunk->article_content : '');
2156		- $content_preview = mb_strlen($content) > $preview_length
2157		- ? mb_substr($content, 0, $preview_length) . '...'
2158		- : $content;
2159		- ?>
2160		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
2161		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
2162		- data-source="<?php echo esc_attr($data_source); ?>"
2163		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
2164		- <td style="padding: 12px 16px; text-align: center;">
2165		- <!-- Checkbox column placeholder for chunks (managed by group) -->
2166		- </td>
2167		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
2168		- <!-- Hidden ID column for chunks -->
2169		- </td>
2170		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2171		- <div class="mxchat-accordion-wrapper">
2172		- <div class="mxchat-content-preview">
2173		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
2174		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
2175		- </span>
2176		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2177		- <?php if (mb_strlen($content) > $preview_length) : ?>
2178		- <button class="mxchat-expand-toggle" type="button">
2179		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2180		- </button>
2181		- <?php endif; ?>
2182		- </div>
2183		- <div class="mxchat-content-full" style="display: none;">
2184		- <div class="content-view">
2185		- <?php
2186		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2187		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2188		- echo wp_kses_post(wpautop($content));
2189		- echo '</div>';
2190		- } else {
2191		- echo wp_kses_post(wpautop($content));
2192		- }
2193		- ?>
2194		- </div>
2195		- </div>
2196		- </div>
2197		- </td>
2198		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2199		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
2200		- </td>
2201		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
2202		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
2203		- </td>
2204		- </tr>
2205		- <?php
2206		- }
2207		- } else {
2208		- // Single entry - display normally with accordion
2209		- $prompt = $first_prompt;
2210		- $content = isset($prompt->display_content) ? $prompt->display_content : (isset($prompt->article_content) ? $prompt->article_content : '');
2211		- $content_preview = mb_strlen($content) > $preview_length
2212		- ? mb_substr($content, 0, $preview_length) . '...'
2213		- : $content;
2214		- ?>
2215		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
2216		- data-source="<?php echo esc_attr($data_source); ?>"
2217		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2218		- <td style="padding: 12px 16px; text-align: center;">
2219		- <input type="checkbox"
2220		- class="mxchat-entry-checkbox"
2221		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2222		- data-source="<?php echo esc_attr($data_source); ?>"
2223		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
2224		- data-is-group="false"
2225		- data-chunk-count="1">
2226		- </td>
2227		- <td style="padding: 12px 16px; font-size: 13px;">
2228		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2229		- </td>
2230		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2231		- <div class="mxchat-accordion-wrapper">
2232		- <div class="mxchat-content-preview">
2233		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2234		- <?php if (mb_strlen($content) > $preview_length) : ?>
2235		- <button class="mxchat-expand-toggle" type="button">
2236		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2237		- </button>
2238		- <?php endif; ?>
2239		- </div>
2240		- <div class="mxchat-content-full" style="display: none;">
2241		- <div class="content-view">
2242		- <?php
2243		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2244		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2245		- echo wp_kses_post(wpautop($content));
2246		- echo '</div>';
2247		- } else {
2248		- echo wp_kses_post(wpautop($content));
2249		- }
2250		- ?>
2251		- </div>
2252		- </div>
2253		- </div>
2254		- </td>
2255		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2256		- <?php
2257		- $actual_source = $source_url;
2258		- if (strpos($source_url, '_ungrouped_') === 0) {
2259		- $actual_source = $prompt->source_url ?? '';
2260		- }
2261		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
2262		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2263		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2264		- <?php esc_html_e('View', 'mxchat'); ?>
2265		- </a>
2266		- <?php else : ?>
2267		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
2268		- <?php endif; ?>
2269		- </td>
2270		- <td style="padding: 12px 16px;">
2271		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-ajax" data-vector-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_pinecone_prompt_nonce'); ?>" style="color: var(--mxch-error);">
2272		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2273		- </button>
2274		- </td>
2275		- </tr>
2276		- <?php
2277		- }
2278		- }
2279		- }
2280		- $html = ob_get_clean();
2281		-
2282		- // Generate pagination HTML for Pinecone
2283		- $total_pages = ceil($total_records / $per_page);
2284		- $pagination_html = '';
2285		- if ($total_pages > 1) {
2286		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
2287		-
2288		- // Previous button
2289		- if ($page > 1) {
2290		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
2291		- }
2292		-
2293		- // Page numbers
2294		- $start_page = max(1, $page - 2);
2295		- $end_page = min($total_pages, $page + 2);
2296		-
2297		- if ($start_page > 1) {
2298		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
2299		- if ($start_page > 2) {
2300		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2301		- }
2302		- }
2303		-
2304		- for ($i = $start_page; $i <= $end_page; $i++) {
2305		- if ($i == $page) {
2306		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
2307		- } else {
2308		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
2309		- }
2310		- }
2311		-
2312		- if ($end_page < $total_pages) {
2313		- if ($end_page < $total_pages - 1) {
2314		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2315		- }
2316		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
2317		- }
2318		-
2319		- // Next button
2320		- if ($page < $total_pages) {
2321		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
2322		- }
2323		-
2324		- $pagination_html .= '</div>';
2325		- }
2326		-
2327		- wp_send_json_success(array(
2328		- 'html' => $html,
2329		- 'pagination_html' => $pagination_html,
2330		- 'total_count' => $total_records,
2331		- 'total_pages' => $total_pages,
2332		- 'page' => $page,
2333		- 'per_page' => $per_page,
2334		- 'data_source' => 'pinecone'
2335		- ));
2336		-}
2337		-
2338		-/**
2339		- * AJAX handler for pagination - handles both WordPress DB and Pinecone data sources
2340		- * Returns paginated entries without requiring a full page reload
2341		- */
2342		-public function ajax_mxchat_paginate_entries() {
2343		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2344		-
2345		- if (!current_user_can('manage_options')) {
2346		- wp_send_json_error(array('message' => 'Unauthorized'));
2347		- return;
2348		- }
2349		-
2350		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2351		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2352		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2353		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2354		- $per_page = 25;
2355		-
2356		- // Check if Pinecone is enabled for this bot
2357		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
2358		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
2359		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2360		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
2361		-
2362		- if ($use_pinecone && $has_pinecone_api) {
2363		- // Delegate to Pinecone pagination handler (pass search params)
2364		- $_POST['page'] = $page;
2365		- $_POST['search'] = $search_query;
2366		- $_POST['content_type'] = $content_type_filter;
2367		- $this->ajax_mxchat_refresh_pinecone_entries();
2368		- return;
2369		- }
2370		-
2371		- // WordPress DB pagination - MUST match initial page load logic exactly
2372		- global $wpdb;
2373		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2374		- $offset = ($page - 1) * $per_page;
2375		-
2376		- // Build WHERE clause for search and content type filtering
2377		- $where_clauses = array();
2378		- $where_values = array();
2379		-
2380		- if ($search_query) {
2381		- $where_clauses[] = "article_content LIKE %s";
2382		- $where_values[] = '%' . $wpdb->esc_like($search_query) . '%';
2383		- }
2384		-
2385		- if ($content_type_filter) {
2386		- switch ($content_type_filter) {
2387		- case 'manual':
2388		- $where_clauses[] = "(source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')";
2389		- break;
2390		- case 'pdf':
2391		- $where_clauses[] = "source_url LIKE '%.pdf'";
2392		- break;
2393		- case 'url':
2394		- $where_clauses[] = "source_url != '' AND source_url IS NOT NULL AND source_url NOT LIKE 'mxchat://%' AND source_url NOT LIKE '%.pdf'";
2395		- break;
2396		- }
2397		- }
2398		-
2399		- $where_sql = !empty($where_clauses) ? 'WHERE ' . implode(' AND ', $where_clauses) : '';
2400		-
2401		- // Count grouped entries with filters applied
2402		- if (!empty($where_values)) {
2403		- $count_args = array_merge($where_values, $where_values);
2404		- $count_query = $wpdb->prepare(
2405		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2406		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))",
2407		- ...$count_args
2408		- );
2409		- $total_records = $wpdb->get_var($count_query);
2410		- } else if (!empty($where_sql)) {
2411		- // Content type filter only (no search), no prepared values needed
2412		- $total_records = $wpdb->get_var(
2413		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2414		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))"
2415		- );
2416		- } else {
2417		- // No filters
2418		- $total_records = $wpdb->get_var(
2419		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2420		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
2421		- );
2422		- }
2423		- $total_pages = ceil($total_records / $per_page);
2424		-
2425		- // Step 1: Get unique source_urls for this page (ordered by latest timestamp) with filters
2426		- if (!empty($where_values)) {
2427		- $query_args = array_merge($where_values, array($per_page, $offset));
2428		- $urls_query = $wpdb->prepare(
2429		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2430		- {$where_sql}
2431		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2432		- ...$query_args
2433		- );
2434		- } else if (!empty($where_sql)) {
2435		- $urls_query = $wpdb->prepare(
2436		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2437		- {$where_sql}
2438		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2439		- $per_page, $offset
2440		- );
2441		- } else {
2442		- $urls_query = $wpdb->prepare(
2443		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2444		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2445		- $per_page, $offset
2446		- );
2447		- }
2448		- $page_urls = $wpdb->get_results($urls_query);
2449		-
2450		- // Step 2: Build list of source_urls to fetch
2451		- $url_list = array();
2452		- $url_order_map = array();
2453		- $order_index = 0;
2454		- foreach ($page_urls as $url_row) {
2455		- $url = $url_row->source_url;
2456		- $url_list[] = $url;
2457		- $url_order_map[$url] = $order_index++;
2458		- }
2459		-
2460		- // Step 3: Fetch all rows for these source_urls (with search filter if applicable)
2461		- $prompts = array();
2462		- if (!empty($url_list)) {
2463		- $placeholders = implode(',', array_fill(0, count($url_list), '%s'));
2464		- if ($search_query) {
2465		- // Include search filter in the final fetch
2466		- $prompts_query = $wpdb->prepare(
2467		- "SELECT id, article_content, source_url, timestamp, role_restriction
2468		- FROM {$table_name}
2469		- WHERE source_url IN ($placeholders) AND article_content LIKE %s
2470		- ORDER BY timestamp DESC",
2471		- ...array_merge($url_list, ['%' . $wpdb->esc_like($search_query) . '%'])
2472		- );
2473		- } else {
2474		- $prompts_query = $wpdb->prepare(
2475		- "SELECT id, article_content, source_url, timestamp, role_restriction
2476		- FROM {$table_name}
2477		- WHERE source_url IN ($placeholders)
2478		- ORDER BY timestamp DESC",
2479		- $url_list
2480		- );
2481		- }
2482		- $prompts = $wpdb->get_results($prompts_query);
2483		- }
2484		-
2485		- // Group prompts by source_url for chunk display
2486		- $grouped_prompts = array();
2487		- foreach ($prompts as $prompt) {
2488		- $source_url = $prompt->source_url ?? '';
2489		-
2490		- // Parse chunk metadata using the proper chunker method (same as initial page load)
2491		- if (class_exists('MxChat_Chunker')) {
2492		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($prompt->article_content);
2493		- $prompt->chunk_metadata = $chunk_meta['metadata'];
2494		- $prompt->display_content = $chunk_meta['text'];
2495		- } else {
2496		- $prompt->chunk_metadata = array();
2497		- $prompt->display_content = $prompt->article_content;
2498		- }
2499		-
2500		- if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0) {
2501		- if (!isset($grouped_prompts[$source_url])) {
2502		- $grouped_prompts[$source_url] = array();
2503		- }
2504		- $grouped_prompts[$source_url][] = $prompt;
2505		- } else {
2506		- // Ungrouped entries
2507		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2508		- }
2509		- }
2510		-
2511		- // Sort groups by the original URL order (newest first)
2512		- uksort($grouped_prompts, function($a, $b) use ($url_order_map) {
2513		- $order_a = isset($url_order_map[$a]) ? $url_order_map[$a] : PHP_INT_MAX;
2514		- $order_b = isset($url_order_map[$b]) ? $url_order_map[$b] : PHP_INT_MAX;
2515		- return $order_a - $order_b;
2516		- });
2517		-
2518		- // Sort each group internally by chunk_index
2519		- foreach ($grouped_prompts as $source_url => &$group) {
2520		- usort($group, function($a, $b) {
2521		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2522		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2523		- return $index_a - $index_b;
2524		- });
2525		- }
2526		- unset($group);
2527		-
2528		- // Build HTML for the table rows
2529		- ob_start();
2530		- $display_index = 0;
2531		- $current_page = $page;
2532		- $data_source = 'wordpress';
2533		- $current_bot_id = $bot_id;
2534		- $preview_length = 150;
2535		-
2536		- if (empty($grouped_prompts)) {
2537		- echo '<tr><td colspan="5" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2538		- esc_html_e('No knowledge entries found. Use the Import Options to add content.', 'mxchat');
2539		- echo '</td></tr>';
2540		- } else {
2541		- foreach ($grouped_prompts as $source_url => $group) {
2542		- $chunk_count = count($group);
2543		- $first_prompt = $group[0];
2544		- $display_index++;
2545		-
2546		- if ($chunk_count > 1) {
2547		- // Multiple chunks - show grouped row with expand button
2548		- $group_id = 'group-' . md5($source_url);
2549		- ?>
2550		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2551		- class="mxchat-chunk-group-header"
2552		- data-source="<?php echo esc_attr($data_source); ?>"
2553		- data-group-id="<?php echo esc_attr($group_id); ?>"
2554		- style="border-bottom: 1px solid var(--mxch-card-border);">
2555		- <td style="padding: 12px 16px; text-align: center;">
2556		- <input type="checkbox"
2557		- class="mxchat-entry-checkbox"
2558		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2559		- data-source="<?php echo esc_attr($data_source); ?>"
2560		- data-source-url="<?php echo esc_attr($source_url); ?>"
2561		- data-is-group="true"
2562		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2563		- </td>
2564		- <td style="padding: 12px 16px; font-size: 13px;">
2565		- <?php echo esc_html($first_prompt->id); ?>
2566		- </td>
2567		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2568		- <div class="mxchat-chunk-group-info">
2569		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2570		- <span class="dashicons dashicons-arrow-right-alt2"></span>
2571		- </button>
2572		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2573		- <span class="mxchat-chunk-preview">
2574		- <?php
2575		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : $first_prompt->article_content;
2576		- $content_preview = mb_substr($parent_content, 0, 100);
2577		- echo esc_html($content_preview . '...');
2578		- ?>
2579		- </span>
2580		- </div>
2581		- </td>
2582		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2583		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2584		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2585		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2586		- <?php esc_html_e('View Source', 'mxchat'); ?>
2587		- </a>
2588		- <?php else : ?>
2589		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2590		- <?php endif; ?>
2591		- </td>
2592		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2593		- <?php if ($data_source !== 'pinecone') : ?>
2594		- <button type="button"
2595		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2596		- data-source-url="<?php echo esc_attr($source_url); ?>"
2597		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2598		- data-data-source="<?php echo esc_attr($data_source); ?>"
2599		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2600		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2601		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2602		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2603		- </button>
2604		- <?php endif; ?>
2605		- <button type="button"
2606		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2607		- data-source-url="<?php echo esc_attr($source_url); ?>"
2608		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2609		- data-data-source="<?php echo esc_attr($data_source); ?>"
2610		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2611		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2612		- style="color: var(--mxch-error);"
2613		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2614		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2615		- </button>
2616		- </td>
2617		- </tr>
2618		- <?php
2619		- // Render hidden chunk rows
2620		- foreach ($group as $chunk_index => $chunk) {
2621		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
2622		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
2623		- $content = isset($chunk->display_content) ? $chunk->display_content : $chunk->article_content;
2624		- $content_preview = mb_strlen($content) > $preview_length
2625		- ? mb_substr($content, 0, $preview_length) . '...'
2626		- : $content;
2627		- ?>
2628		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
2629		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
2630		- data-source="<?php echo esc_attr($data_source); ?>"
2631		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
2632		- <td style="padding: 12px 16px; text-align: center;">
2633		- <!-- Checkbox column placeholder for chunks (managed by group) -->
2634		- </td>
2635		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
2636		- <!-- Hidden ID column for chunks -->
2637		- </td>
2638		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2639		- <div class="mxchat-accordion-wrapper">
2640		- <div class="mxchat-content-preview">
2641		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
2642		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
2643		- </span>
2644		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2645		- <?php if (mb_strlen($content) > $preview_length) : ?>
2646		- <button class="mxchat-expand-toggle" type="button">
2647		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2648		- </button>
2649		- <?php endif; ?>
2650		- </div>
2651		- <div class="mxchat-content-full" style="display: none;">
2652		- <div class="content-view">
2653		- <?php
2654		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2655		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2656		- echo wp_kses_post(wpautop($content));
2657		- echo '</div>';
2658		- } else {
2659		- echo wp_kses_post(wpautop($content));
2660		- }
2661		- ?>
2662		- </div>
2663		- </div>
2664		- </div>
2665		- </td>
2666		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2667		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
2668		- </td>
2669		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
2670		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
2671		- </td>
2672		- </tr>
2673		- <?php
2674		- }
2675		- } else {
2676		- // Single entry - display normally with accordion
2677		- $prompt = $first_prompt;
2678		- $content = isset($prompt->display_content) ? $prompt->display_content : $prompt->article_content;
2679		- $content_preview = mb_strlen($content) > $preview_length
2680		- ? mb_substr($content, 0, $preview_length) . '...'
2681		- : $content;
2682		- ?>
2683		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
2684		- data-source="<?php echo esc_attr($data_source); ?>"
2685		- style="border-bottom: 1px solid var(--mxch-card-border);">
2686		- <td style="padding: 12px 16px; text-align: center;">
2687		- <input type="checkbox"
2688		- class="mxchat-entry-checkbox"
2689		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2690		- data-source="<?php echo esc_attr($data_source); ?>"
2691		- data-source-url="<?php echo esc_attr($source_url); ?>"
2692		- data-is-group="false">
2693		- </td>
2694		- <td style="padding: 12px 16px; font-size: 13px;">
2695		- <?php echo esc_html($prompt->id); ?>
2696		- </td>
2697		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2698		- <div class="mxchat-accordion-wrapper">
2699		- <div class="mxchat-content-preview">
2700		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2701		- <?php if (mb_strlen($content) > $preview_length) : ?>
2702		- <button class="mxchat-expand-toggle" type="button">
2703		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2704		- </button>
2705		- <?php endif; ?>
2706		- </div>
2707		- <div class="mxchat-content-full" style="display: none;">
2708		- <div class="content-view">
2709		- <?php
2710		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2711		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2712		- echo wp_kses_post(wpautop($content));
2713		- echo '</div>';
2714		- } else {
2715		- echo wp_kses_post(wpautop($content));
2716		- }
2717		- ?>
2718		- </div>
2719		- </div>
2720		- </div>
2721		- </td>
2722		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2723		- <?php
2724		- $actual_source = $source_url;
2725		- if (strpos($source_url, '_ungrouped_') === 0) {
2726		- $actual_source = $prompt->source_url ?? '';
2727		- }
2728		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
2729		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2730		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2731		- <?php esc_html_e('View', 'mxchat'); ?>
2732		- </a>
2733		- <?php else : ?>
2734		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
2735		- <?php endif; ?>
2736		- </td>
2737		- <td style="padding: 12px 16px; white-space: nowrap;">
2738		- <button type="button"
2739		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2740		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
2741		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2742		- data-data-source="<?php echo esc_attr($data_source); ?>"
2743		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2744		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2745		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2746		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2747		- </button>
2748		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-wordpress" data-entry-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_wordpress_prompt_nonce'); ?>" style="color: var(--mxch-error);">
2749		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2750		- </button>
2751		- </td>
2752		- </tr>
2753		- <?php
2754		- }
2755		- }
2756		- }
2757		- $html = ob_get_clean();
2758		-
2759		- // Generate pagination HTML (include search/filter data for subsequent pages)
2760		- $pagination_html = '';
2761		- if ($total_pages > 1) {
2762		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
2763		-
2764		- // Previous button
2765		- if ($page > 1) {
2766		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
2767		- }
2768		-
2769		- // Page numbers
2770		- $start_page = max(1, $page - 2);
2771		- $end_page = min($total_pages, $page + 2);
2772		-
2773		- if ($start_page > 1) {
2774		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
2775		- if ($start_page > 2) {
2776		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2777		- }
2778		- }
2779		-
2780		- for ($i = $start_page; $i <= $end_page; $i++) {
2781		- if ($i == $page) {
2782		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
2783		- } else {
2784		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
2785		- }
2786		- }
2787		-
2788		- if ($end_page < $total_pages) {
2789		- if ($end_page < $total_pages - 1) {
2790		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2791		- }
2792		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
2793		- }
2794		-
2795		- // Next button
2796		- if ($page < $total_pages) {
2797		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
2798		- }
2799		-
2800		- $pagination_html .= '</div>';
2801		- }
2802		-
2803		- wp_send_json_success(array(
2804		- 'html' => $html,
2805		- 'pagination_html' => $pagination_html,
2806		- 'total_count' => $total_records,
2807		- 'total_pages' => $total_pages,
2808		- 'page' => $page,
2809		- 'per_page' => $per_page,
2810		- 'data_source' => 'wordpress'
2811		- ));
2812		-}
2813		-
2814		-/**
2815		- * AJAX handler to detect available sitemaps on the site
2816		- * Optimized for speed - only checks primary sitemap indexes first
2817		- */
2818		-public function ajax_mxchat_detect_sitemaps() {
2819		- check_ajax_referer('mxchat_detect_sitemaps_nonce', 'nonce');
2820		-
2821		- if (!current_user_can('manage_options')) {
2822		- wp_send_json_error(array('message' => 'Unauthorized'));
2823		- return;
2824		- }
2825		-
2826		- $site_url = get_site_url();
2827		- $sitemaps = array();
2828		- $found_index = false;
2829		-
2830		- // Only check the main sitemap index files first (much faster)
2831		- // These are the primary entry points that contain sub-sitemaps
2832		- $primary_indexes = array(
2833		- 'sitemap_index.xml' => 'Yoast SEO / Rank Math', // Yoast & Rank Math
2834		- 'wp-sitemap.xml' => 'WordPress Core', // WordPress Core
2835		- 'sitemap.xml' => 'Standard', // Generic/AIOSEO
2836		- );
2837		-
2838		- foreach ($primary_indexes as $path => $source) {
2839		- $url = trailingslashit($site_url) . $path;
2840		-
2841		- $response = wp_remote_head($url, array(
2842		- 'timeout' => 10,
2843		- 'sslverify' => false,
2844		- 'redirection' => 1,
2845		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2846		- ));
2847		-
2848		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2849		- // Found a sitemap index - parse it to get sub-sitemaps
2850		- $sub_sitemaps = $this->parse_sitemap_index($url);
2851		- if (!empty($sub_sitemaps)) {
2852		- $sitemaps[] = array(
2853		- 'url' => $url,
2854		- 'type' => 'index',
2855		- 'source' => $source,
2856		- 'sub_sitemaps' => $sub_sitemaps
2857		- );
2858		- $found_index = true;
2859		- // Found a valid index, no need to check others
2860		- break;
2861		- }
2862		- }
2863		- }
2864		-
2865		- // If no sitemap index found, check for standalone sitemaps
2866		- if (!$found_index) {
2867		- $standalone_sitemaps = array(
2868		- 'post-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
2869		- 'page-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
2870		- );
2871		-
2872		- foreach ($standalone_sitemaps as $path => $info) {
2873		- $url = trailingslashit($site_url) . $path;
2874		-
2875		- $response = wp_remote_head($url, array(
2876		- 'timeout' => 2,
2877		- 'sslverify' => false
2878		- ));
2879		-
2880		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2881		- $sitemaps[] = array(
2882		- 'url' => $url,
2883		- 'type' => $info['type'],
2884		- 'source' => $info['source'],
2885		- 'url_count' => 0 // Skip URL count for speed
2886		- );
2887		- }
2888		- }
2889		- }
2890		-
2891		- wp_send_json_success(array(
2892		- 'sitemaps' => $sitemaps,
2893		- 'site_url' => $site_url
2894		- ));
2895		-}
2896		-
2897		-/**
2898		- * Parse a sitemap index to get sub-sitemaps
2899		- * Optimized: doesn't fetch URL count for each sub-sitemap (too slow)
2900		- */
2901		-private function parse_sitemap_index($url) {
2902		- $sub_sitemaps = array();
2903		-
2904		- $response = wp_remote_get($url, array(
2905		- 'timeout' => 30,
2906		- 'sslverify' => false,
2907		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2908		- 'headers' => array(
2909		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
2910		- 'Accept-Language' => 'en-US,en;q=0.9',
2911		- ),
2912		- ));
2913		-
2914		- if (is_wp_error($response)) {
2915		- return $sub_sitemaps;
2916		- }
2917		-
2918		- $body = wp_remote_retrieve_body($response);
2919		- if (empty($body)) {
2920		- return $sub_sitemaps;
2921		- }
2922		-
2923		- // Suppress XML errors
2924		- libxml_use_internal_errors(true);
2925		- $xml = simplexml_load_string($body);
2926		- libxml_clear_errors();
2927		-
2928		- if ($xml === false) {
2929		- return $sub_sitemaps;
2930		- }
2931		-
2932		- // Check if it's a sitemap index (contains <sitemap> elements)
2933		- if (isset($xml->sitemap)) {
2934		- foreach ($xml->sitemap as $sitemap) {
2935		- $loc = (string) $sitemap->loc;
2936		- if (!empty($loc)) {
2937		- // Try to determine the type from the URL
2938		- $type = 'content';
2939		- if (strpos($loc, 'category') !== false || strpos($loc, 'tag') !== false || strpos($loc, 'taxonomy') !== false) {
2940		- $type = 'taxonomy';
2941		- } elseif (strpos($loc, 'author') !== false || strpos($loc, 'user') !== false) {
2942		- $type = 'author';
2943		- }
2944		-
2945		- // Skip URL count - too slow to fetch for each sitemap
2946		- $sub_sitemaps[] = array(
2947		- 'url' => $loc,
2948		- 'type' => $type,
2949		- 'url_count' => 0, // Don't fetch - takes too long
2950		- 'name' => basename(parse_url($loc, PHP_URL_PATH))
2951		- );
2952		- }
2953		- }
2954		- }
2955		-
2956		- return $sub_sitemaps;
2957		-}
2958		-
2959		-/**
2960		- * Get URL count from a sitemap
2961		- */
2962		-private function get_sitemap_url_count($url) {
2963		- $response = wp_remote_get($url, array(
2964		- 'timeout' => 30,
2965		- 'sslverify' => false,
2966		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2967		- 'headers' => array(
2968		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
2969		- 'Accept-Language' => 'en-US,en;q=0.9',
2970		- ),
2971		- ));
2972		-
2973		- if (is_wp_error($response)) {
2974		- return 0;
2975		- }
2976		-
2977		- $body = wp_remote_retrieve_body($response);
2978		- if (empty($body)) {
2979		- return 0;
2980		- }
2981		-
2982		- // Count <url> or <loc> elements
2983		- $count = preg_match_all('/<url>/i', $body, $matches);
2984		- return $count ?: 0;
2985		-}
2986		-
2987		-/**
2988		- * Get sitemaps declared in robots.txt
2989		- */
2990		-private function get_sitemaps_from_robots($site_url) {
2991		- $sitemaps = array();
2992		- $robots_url = trailingslashit($site_url) . 'robots.txt';
2993		-
2994		- $response = wp_remote_get($robots_url, array(
2995		- 'timeout' => 15,
2996		- 'sslverify' => false,
2997		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2998		- ));
2999		-
3000		- if (is_wp_error($response)) {
3001		- return $sitemaps;
3002		- }
3003		-
3004		- $body = wp_remote_retrieve_body($response);
3005		- if (empty($body)) {
3006		- return $sitemaps;
3007		- }
3008		-
3009		- // Find Sitemap: declarations
3010		- if (preg_match_all('/^Sitemap:\s*(.+)$/mi', $body, $matches)) {
3011		- foreach ($matches[1] as $sitemap_url) {
3012		- $sitemap_url = trim($sitemap_url);
3013		- if (filter_var($sitemap_url, FILTER_VALIDATE_URL)) {
3014		- $sitemaps[] = $sitemap_url;
3015		- }
3016		- }
3017		- }
3018		-
3019		- return $sitemaps;
3020		-}
3021		-
3022	1767	public function mxchat_stop_processing() {
3023	1768	// Verify permissions
3024	1769	if (!current_user_can('manage_options')) {
3025	1770	wp_die(esc_html__('Unauthorized access', 'mxchat'));
		@@ -3027,52 +1772,28 @@
3027	1772
3028	1773	// Verify nonce
3029	1774	check_admin_referer('mxchat_stop_processing_action', 'mxchat_stop_processing_nonce');
3030	1775
3031		- global $wpdb;
3032		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
3033		-
3034		- // Get active queue IDs
3035		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
3036		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
3037		-
3038		- // Delete all pending items from active queues
3039		- if ($sitemap_queue_id) {
3040		- $wpdb->delete(
3041		- $table_name,
3042		- array(
3043		- 'queue_id' => $sitemap_queue_id,
3044		- 'status' => 'pending'
3045		- ),
3046		- array('%s', '%s')
3047		- );
3048		-
3049		- delete_transient('mxchat_active_queue_sitemap');
	1776	+ // Get the last sitemap URL and clear its transient
	1777	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
	1778	+ if ($sitemap_url) {
	1779	+ delete_transient('mxchat_sitemap_status_' . md5($sitemap_url));
3050	1780	delete_transient('mxchat_last_sitemap_url');
3051	1781	}
3052		-
3053		- if ($pdf_queue_id) {
3054		- // Get PDF path before deleting
3055		- $pdf_path = $this->mxchat_get_queue_meta($pdf_queue_id, 'pdf_path');
3056		-
3057		- $wpdb->delete(
3058		- $table_name,
3059		- array(
3060		- 'queue_id' => $pdf_queue_id,
3061		- 'status' => 'pending'
3062		- ),
3063		- array('%s', '%s')
3064		- );
3065		-
3066		- // Delete PDF file
3067		- if ($pdf_path && file_exists($pdf_path)) {
3068		- wp_delete_file($pdf_path);
3069		- }
3070		-
3071		- delete_transient('mxchat_active_queue_pdf');
	1782	+
	1783	+ // Get the last PDF URL and clear its transient
	1784	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1785	+ if ($pdf_url) {
	1786	+ delete_transient('mxchat_pdf_status_' . md5($pdf_url));
3072	1787	delete_transient('mxchat_last_pdf_url');
3073	1788	}
3074	1789
	1790	+ // Unschedule any pending sitemap events
	1791	+ $timestamp = wp_next_scheduled('mxchat_process_sitemap_urls');
	1792	+ if ($timestamp) {
	1793	+ wp_unschedule_event($timestamp, 'mxchat_process_sitemap_urls');
	1794	+ }
	1795	+
3075	1796	// Redirect back with a success message
3076	1797	set_transient('mxchat_admin_notice_success',
3077	1798	esc_html__('Processing has been stopped successfully.', 'mxchat'),
3078	1799	30
		@@ -3079,12 +1800,8 @@
3079	1800	);
3080	1801	wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
3081	1802	exit;
3082	1803	}
3083		-
3084		-/**
3085		- * Get content list for processing
3086		- */
3087	1804	public function ajax_mxchat_get_content_list() {
3088	1805	// Verify the nonce
3089	1806	check_ajax_referer('mxchat_content_selector_nonce', 'nonce');
3090	1807
		@@ -3092,9 +1809,9 @@
3092	1809	wp_send_json_error(__('Unauthorized access', 'mxchat'));
3093	1810	}
3094	1811
3095	1812	$page = isset($_GET['page']) ? absint($_GET['page']) : 1;
3096		- $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 100;
	1813	+ $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 20;
3097	1814	$search = isset($_GET['search']) ? sanitize_text_field($_GET['search']) : '';
3098	1815	$post_type = isset($_GET['post_type']) ? sanitize_text_field($_GET['post_type']) : 'all';
3099	1816	$post_status = isset($_GET['post_status']) ? sanitize_text_field($_GET['post_status']) : 'publish';
3100	1817	$processed_filter = isset($_GET['processed_filter']) ? sanitize_text_field($_GET['processed_filter']) : 'all';
		@@ -3107,47 +1824,20 @@
3107	1824	'orderby' => 'date',
3108	1825	'order' => 'DESC',
3109	1826	);
3110	1827
3111		- // Handle post types - IMPROVED VERSION
	1828	+ // Handle post types
3112	1829	if ($post_type !== 'all') {
3113	1830	$args['post_type'] = $post_type;
3114	1831	} else {
3115		- // Get all available post types that might contain content
3116		- $all_post_types = array();
	1832	+ // Default to post and page if we can't get post types
	1833	+ $args['post_type'] = array('post', 'page');
3117	1834
3118		- // First get all public post types
3119		- $public_types = get_post_types(array('public' => true), 'names');
3120		- $all_post_types = array_merge($all_post_types, $public_types);
3121		-
3122		- // Add common forum/community post types
3123		- $forum_types = array('topic', 'reply', 'forum', 'wpforo_topic', 'wpforo_post');
3124		- foreach ($forum_types as $forum_type) {
3125		- if (post_type_exists($forum_type)) {
3126		- $all_post_types[] = $forum_type;
3127		- }
	1835	+ // Try to get public post types
	1836	+ $public_types = $this->mxchat_get_public_post_types();
	1837	+ if (is_array($public_types) && !empty($public_types)) {
	1838	+ $args['post_type'] = array_keys($public_types);
3128	1839	}
3129		-
3130		- // Add other commonly used post types
3131		- $common_types = array('product', 'job_listing', 'event', 'portfolio');
3132		- foreach ($common_types as $common_type) {
3133		- if (post_type_exists($common_type)) {
3134		- $all_post_types[] = $common_type;
3135		- }
3136		- }
3137		-
3138		- // Remove duplicates and ensure we have at least some post types
3139		- $all_post_types = array_unique($all_post_types);
3140		-
3141		- if (empty($all_post_types)) {
3142		- // Fallback to basic post types
3143		- $all_post_types = array('post', 'page');
3144		- }
3145		-
3146		- $args['post_type'] = $all_post_types;
3147		-
3148		- // Debug logging to see what post types are being queried
3149		- //error_log('MxChat Debug: Querying post types: ' . implode(', ', $all_post_types));
3150	1840	}
3151	1841
3152	1842	if (!empty($search)) {
3153	1843	$args['s'] = $search;
		@@ -3152,58 +1842,36 @@
3152	1842	if (!empty($search)) {
3153	1843	$args['s'] = $search;
3154	1844	}
3155	1845
3156		- // Get processed data from storage
	1846	+ // ================================
	1847	+ // FIXED: Check only the ACTIVE storage method
	1848	+ // ================================
	1849	+
3157	1850	$processed_data = array();
3158	1851
	1852	+ // Check if Pinecone is enabled
3159	1853	$pinecone_options = get_option('mxchat_pinecone_addon_options', array());
3160	1854	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3161		-
	1855	+
3162	1856	if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3163		- // Get fresh data from Pinecone - no caching
	1857	+ // ONLY check Pinecone if it's enabled
3164	1858	$processed_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
3165	1859	} else {
3166		- // WordPress DB checking with better URL matching for all post types
	1860	+ // ONLY check WordPress DB if Pinecone is not enabled
3167	1861	global $wpdb;
3168	1862	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3169		- $processed_items = $wpdb->get_results("SELECT id, source_url, article_content, timestamp FROM {$table_name}");
3170		-
3171		- // Group items by source_url to count chunks
3172		- $url_chunk_counts = array();
3173		- $url_latest_timestamp = array();
3174		- $url_first_id = array();
3175		-
	1863	+ $processed_items = $wpdb->get_results("SELECT id, source_url, timestamp FROM {$table_name}");
	1864	+
3176	1865	if (!empty($processed_items)) {
3177	1866	foreach ($processed_items as $item) {
3178		- $url = $item->source_url;
3179		- if (empty($url)) continue;
3180		-
3181		- // Count chunks per URL
3182		- if (!isset($url_chunk_counts[$url])) {
3183		- $url_chunk_counts[$url] = 0;
3184		- $url_latest_timestamp[$url] = $item->timestamp;
3185		- $url_first_id[$url] = $item->id;
3186		- }
3187		- $url_chunk_counts[$url]++;
3188		-
3189		- // Track latest timestamp
3190		- if (strtotime($item->timestamp) > strtotime($url_latest_timestamp[$url])) {
3191		- $url_latest_timestamp[$url] = $item->timestamp;
3192		- }
3193		- }
3194		-
3195		- // Now build processed_data with chunk counts
3196		- foreach ($url_chunk_counts as $url => $chunk_count) {
3197		- $post_id = $this->mxchat_url_to_post_id_improved($url);
3198		-
	1867	+ $post_id = url_to_postid($item->source_url);
3199	1868	if ($post_id) {
3200	1869	$processed_data[$post_id] = array(
3201		- 'db_id' => $url_first_id[$url],
3202		- 'timestamp' => $url_latest_timestamp[$url],
3203		- 'url' => $url,
3204		- 'source' => 'wordpress',
3205		- 'chunk_count' => $chunk_count
	1870	+ 'db_id' => $item->id,
	1871	+ 'timestamp' => $item->timestamp,
	1872	+ 'url' => $item->source_url,
	1873	+ 'source' => 'wordpress'
3206	1874	);
3207	1875	}
3208	1876	}
3209	1877	}
		@@ -3208,8 +1876,10 @@
3208	1876	}
3209	1877	}
3210	1878	}
3211	1879
	1880	+ // ================================
	1881	+
3212	1882	// Get processed IDs as a simple array for in_array checks
3213	1883	$processed_ids = array_keys($processed_data);
3214	1884
3215	1885	// Handle processed/unprocessed filter
		@@ -3251,14 +1921,8 @@
3251	1921	$db_record_id = $item_data['db_id'];
3252	1922	}
3253	1923	}
3254	1924
3255		- // Get chunk count for this item
3256		- $chunk_count = 0;
3257		- if ($is_processed && isset($processed_data[$id]['chunk_count'])) {
3258		- $chunk_count = intval($processed_data[$id]['chunk_count']);
3259		- }
3260		-
3261	1925	$content_items[] = array(
3262	1926	'id' => $id,
3263	1927	'title' => get_the_title(),
3264	1928	'permalink' => get_permalink(),
		@@ -3269,10 +1933,9 @@
3269	1933	'word_count' => $word_count,
3270	1934	'already_processed' => $is_processed,
3271	1935	'processed_date' => $processed_date,
3272	1936	'db_record_id' => $db_record_id,
3273		- 'data_source' => $data_source,
3274		- 'chunk_count' => $chunk_count
	1937	+ 'data_source' => $data_source
3275	1938	);
3276	1939	}
3277	1940	wp_reset_postdata();
3278	1941	}
		@@ -3288,178 +1951,8 @@
3288	1951	wp_send_json_success($response);
3289	1952	exit;
3290	1953	}
3291	1954
3292		-
3293		-/**
3294		- * This function handles various WooCommerce URL formats and permalink structures
3295		- */
3296		-private function mxchat_url_to_post_id_improved($url) {
3297		- // First try the standard WordPress function
3298		- $post_id = url_to_postid($url);
3299		-
3300		- if ($post_id > 0) {
3301		- return $post_id;
3302		- }
3303		-
3304		- // If that fails, try more aggressive URL matching
3305		- // Remove trailing slashes and query parameters for better matching
3306		- $clean_url = rtrim($url, '/');
3307		- $clean_url = strtok($clean_url, '?'); // Remove query parameters
3308		-
3309		- // Try again with cleaned URL
3310		- $post_id = url_to_postid($clean_url);
3311		- if ($post_id > 0) {
3312		- return $post_id;
3313		- }
3314		-
3315		- // For bbPress forum topics, try extracting slug from URL
3316		- if (strpos($url, '/topic/') !== false || strpos($url, '/forums/') !== false) {
3317		- // Handle bbPress URLs: /forums/topic/topic-name/
3318		- if (preg_match('/\/forums\/topic\/([^\/\?]+)/', $url, $matches)) {
3319		- $topic_slug = $matches[1];
3320		-
3321		- // Look up topic by slug
3322		- $topic = get_page_by_path($topic_slug, OBJECT, 'topic');
3323		- if ($topic) {
3324		- return $topic->ID;
3325		- }
3326		-
3327		- // Alternative method: query by post_name
3328		- global $wpdb;
3329		- $post_id = $wpdb->get_var($wpdb->prepare(
3330		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3331		- $topic_slug
3332		- ));
3333		-
3334		- if ($post_id) {
3335		- return intval($post_id);
3336		- }
3337		- }
3338		-
3339		- // Handle simpler topic URLs: /topic/topic-name/
3340		- if (preg_match('/\/topic\/([^\/\?]+)/', $url, $matches)) {
3341		- $topic_slug = $matches[1];
3342		-
3343		- global $wpdb;
3344		- $post_id = $wpdb->get_var($wpdb->prepare(
3345		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3346		- $topic_slug
3347		- ));
3348		-
3349		- if ($post_id) {
3350		- return intval($post_id);
3351		- }
3352		- }
3353		- }
3354		-
3355		- // For WooCommerce products
3356		- if (strpos($url, '/product/') !== false || strpos($url, 'product=') !== false) {
3357		- // Extract product slug from various URL formats
3358		- $product_slug = '';
3359		-
3360		- // Handle pretty permalinks: /product/product-name/
3361		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
3362		- $product_slug = $matches[1];
3363		- }
3364		- // Handle query parameters: ?product=product-name
3365		- elseif (preg_match('/[\?&]product=([^&]+)/', $url, $matches)) {
3366		- $product_slug = $matches[1];
3367		- }
3368		-
3369		- if (!empty($product_slug)) {
3370		- // Look up product by slug
3371		- $product = get_page_by_path($product_slug, OBJECT, 'product');
3372		- if ($product) {
3373		- return $product->ID;
3374		- }
3375		-
3376		- // Alternative method: query by post_name
3377		- global $wpdb;
3378		- $post_id = $wpdb->get_var($wpdb->prepare(
3379		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'product' AND post_status = 'publish'",
3380		- $product_slug
3381		- ));
3382		-
3383		- if ($post_id) {
3384		- return intval($post_id);
3385		- }
3386		- }
3387		- }
3388		-
3389		- // Generic approach: try to extract slug and match against all post types
3390		- $parsed_url = wp_parse_url($clean_url);
3391		- $path = $parsed_url['path'] ?? '';
3392		-
3393		- if (!empty($path)) {
3394		- // Get the last part of the path as potential slug
3395		- $path_parts = array_filter(explode('/', trim($path, '/')));
3396		- $potential_slug = end($path_parts);
3397		-
3398		- if (!empty($potential_slug)) {
3399		- global $wpdb;
3400		-
3401		- // Try to find any post with this slug
3402		- $post_id = $wpdb->get_var($wpdb->prepare(
3403		- "SELECT ID FROM {$wpdb->posts}
3404		- WHERE post_name = %s
3405		- AND post_status IN ('publish', 'closed', 'private')
3406		- AND post_type NOT IN ('revision', 'attachment', 'nav_menu_item')
3407		- ORDER BY CASE
3408		- WHEN post_type = 'post' THEN 1
3409		- WHEN post_type = 'page' THEN 2
3410		- WHEN post_type = 'topic' THEN 3
3411		- WHEN post_type = 'product' THEN 4
3412		- ELSE 5
3413		- END
3414		- LIMIT 1",
3415		- $potential_slug
3416		- ));
3417		-
3418		- if ($post_id) {
3419		- return intval($post_id);
3420		- }
3421		- }
3422		- }
3423		-
3424		- // ADDITIONAL: Try direct database lookup by URL variations
3425		- global $wpdb;
3426		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3427		-
3428		- // Try variations of the URL (with/without trailing slash, http/https)
3429		- $url_variations = array(
3430		- $url,
3431		- rtrim($url, '/'),
3432		- $url . '/',
3433		- str_replace('http://', 'https://', $url),
3434		- str_replace('https://', 'http://', $url),
3435		- str_replace('http://', 'https://', rtrim($url, '/')),
3436		- str_replace('https://', 'http://', rtrim($url, '/'))
3437		- );
3438		-
3439		- // Remove duplicates
3440		- $url_variations = array_unique($url_variations);
3441		-
3442		- foreach ($url_variations as $variation) {
3443		- $existing_record = $wpdb->get_row($wpdb->prepare(
3444		- "SELECT id, source_url FROM $table_name WHERE source_url = %s",
3445		- $variation
3446		- ));
3447		-
3448		- if ($existing_record) {
3449		- // Try to get post ID from this stored URL
3450		- $stored_post_id = url_to_postid($existing_record->source_url);
3451		- if ($stored_post_id > 0) {
3452		- return $stored_post_id;
3453		- }
3454		- }
3455		- }
3456		-
3457		- return 0; // No match found
3458		-}
3459		-/**
3460		- * Process selected content via AJAX
3461		- */
3462	1955	public function ajax_mxchat_process_selected_content() {
3463	1956	// Basic request validation
3464	1957	if (!check_ajax_referer('mxchat_content_selector_nonce', 'nonce', false)) {
3465	1958	wp_send_json_error('Invalid nonce');
		@@ -3483,11 +1976,8 @@
3483	1976	wp_send_json_error('No content selected');
3484	1977	exit;
3485	1978	}
3486	1979
3487		- // Get bot_id from request
3488		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
3489		-
3490	1980	// Process only ONE post at a time to avoid request size issues
3491	1981	$post_id = reset($post_ids);
3492	1982	$post = get_post($post_id);
3493	1983
		@@ -3494,151 +1984,15 @@
3494	1984	if (!$post) {
3495	1985	wp_send_json_error('Post not found');
3496	1986	exit;
3497	1987	}
3498		-
3499		- // Allow developers to modify post data before processing into knowledge base
3500		- $post = apply_filters('mxchat_before_process_post', $post, $bot_id);
3501		-
3502		- // Get content including title, short description (for WooCommerce), and main content
3503		- $content = $post->post_title . "\n\n";
3504		-
3505		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
3506		- if (!empty($post->post_excerpt)) {
3507		- // Remove shortcode tags but preserve content inside them
3508		- $clean_excerpt = $this->strip_shortcode_tags_preserve_content($post->post_excerpt);
3509		- $content .= "Short Description: " . wp_strip_all_tags($clean_excerpt) . "\n\n";
3510		- }
3511		-
3512		- // Add main content - remove shortcode tags but preserve content inside them
3513		- $clean_content = $this->strip_shortcode_tags_preserve_content($post->post_content);
3514		- $content .= wp_strip_all_tags($clean_content);
3515		-
3516		- // ADD WOOCOMMERCE PRODUCT DATA (pricing, stock, categories, custom tabs)
3517		- if (get_post_type($post_id) === 'product' && class_exists('WooCommerce')) {
3518		- $product = wc_get_product($post_id);
3519		-
3520		- if ($product) {
3521		- // Get pricing information
3522		- $regular_price = $product->get_regular_price();
3523		- $sale_price = $product->get_sale_price();
3524		- $price = $product->get_price();
3525		- $sku = $product->get_sku();
3526		-
3527		- // Get currency symbol
3528		- $currency_symbol = get_woocommerce_currency_symbol();
3529		-
3530		- // Add pricing information
3531		- $content .= "\n";
3532		- if (!empty($regular_price)) {
3533		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
3534		- } elseif (!empty($price)) {
3535		- $content .= "Price: " . $currency_symbol . $price . "\n";
3536		- }
3537		-
3538		- if (!empty($sale_price) && $sale_price !== $regular_price) {
3539		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
3540		- }
3541		-
3542		- // Handle variable products - show price range
3543		- if ($product->is_type('variable')) {
3544		- $min_price = $product->get_variation_price('min');
3545		- $max_price = $product->get_variation_price('max');
3546		- if ($min_price !== $max_price) {
3547		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
3548		- }
3549		- }
3550		-
3551		- if (!empty($sku)) {
3552		- $content .= "SKU: " . $sku . "\n";
3553		- }
3554		-
3555		- // Get product categories
3556		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
3557		- if (!empty($categories) && !is_wp_error($categories)) {
3558		- $content .= "Categories: " . implode(', ', $categories) . "\n";
3559		- }
3560		- }
3561		-
3562		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
3563		- $custom_tabs = get_post_meta($post_id, 'yikes_woo_products_tabs', true);
3564		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
3565		- foreach ($custom_tabs as $tab) {
3566		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
3567		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
3568		-
3569		- if (!empty($tab_title) && !empty($tab_content)) {
3570		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
3571		- }
3572		- }
3573		- }
3574		-
3575		- // Also check for reusable/saved tabs applied to this product
3576		- $applied_saved_tabs = get_post_meta($post_id, 'yikes_woo_reusable_products_tabs_applied', true);
3577		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
3578		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
3579		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
3580		- foreach ($applied_saved_tabs as $saved_tab_id) {
3581		- if (isset($saved_tabs[$saved_tab_id])) {
3582		- $tab = $saved_tabs[$saved_tab_id];
3583		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
3584		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
3585		-
3586		- if (!empty($tab_title) && !empty($tab_content)) {
3587		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
3588		- }
3589		- }
3590		- }
3591		- }
3592		- }
3593		- }
3594		-
3595		- // ADD ACF FIELDS SUPPORT
3596		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
3597		- if (!empty($acf_fields)) {
3598		- $acf_content_parts = array();
3599		-
3600		- foreach ($acf_fields as $field_name => $field_value) {
3601		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
3602		-
3603		- if (!empty($formatted_value)) {
3604		- $field_label = ucwords(str_replace('_', ' ', $field_name));
3605		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
3606		- }
3607		- }
3608		-
3609		- if (!empty($acf_content_parts)) {
3610		- $content .= "\n\n" . implode("\n", $acf_content_parts);
3611		- }
3612		- }
3613		-
3614		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
3615		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
3616		- if (!empty($custom_meta)) {
3617		- $meta_content_parts = array();
3618		-
3619		- foreach ($custom_meta as $meta_key => $meta_value) {
3620		- // Convert meta key to readable label
3621		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
3622		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
3623		- }
3624		-
3625		- if (!empty($meta_content_parts)) {
3626		- $content .= "\n\n" . implode("\n", $meta_content_parts);
3627		- }
3628		- }
3629		-
3630		- // Debug logging for WordPress Import content
3631		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Post ID: ' . $post_id . ' Title: ' . $post->post_title);
3632		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Raw post_content length: ' . strlen($post->post_content));
3633		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Final content length: ' . strlen($content));
3634		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Content preview: ' . substr($content, 0, 300));
3635		-
3636		- // Note: Removed 10,000 char limit - chunking now handles large content properly
3637		-
3638		- // Get bot-specific API key
3639		- $bot_options = $this->get_bot_options($bot_id);
3640		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	1988	+
	1989	+ // Get minimal content
	1990	+ $content = $post->post_title . "\n\n" . wp_strip_all_tags($post->post_content);
	1991	+ $content = substr($content, 0, 10000); // Limit content size
	1992	+
	1993	+ // Get API key with proper model detection
	1994	+ $options = get_option('mxchat_options');
3641	1995	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
3642	1996
3643	1997	if (strpos($selected_model, 'voyage') === 0) {
3644	1998	$api_key = $options['voyage_api_key'] ?? '';
		@@ -3651,9 +2005,8 @@
3651	2005	$provider_name = 'OpenAI';
3652	2006	}
3653	2007
3654	2008	if (empty($api_key)) {
3655		- MxChat_Admin::mxchat_log_debug('api_error', $provider_name . ' API key not configured for knowledge processing');
3656	2009	wp_send_json_error($provider_name . ' API key not configured');
3657	2010	exit;
3658	2011	}
3659	2012
		@@ -3659,23 +2012,26 @@
3659	2012
3660	2013	$source_url = get_permalink($post_id);
3661	2014	$vector_id = md5($source_url); // Vector ID for Pinecone
3662	2015
3663		- // Check for existing content in bot-specific storage
	2016	+ // ================================
	2017	+ // FIXED: Check for existing content in ONLY the active storage method
	2018	+ // ================================
	2019	+
3664	2020	$is_update = false;
3665	2021
3666		- // Get bot-specific Pinecone configuration
3667		- $bot_pinecone_config = $this->get_bot_pinecone_config($bot_id);
3668		- $use_pinecone = !empty($bot_pinecone_config) && ($bot_pinecone_config['use_pinecone'] ?? false);
	2022	+ // Check if Pinecone is enabled
	2023	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	2024	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3669	2025
3670		- if ($use_pinecone && !empty($bot_pinecone_config['api_key'])) {
3671		- // Check Pinecone for this bot
3672		- $pinecone_data = $this->mxchat_get_pinecone_processed_content($bot_pinecone_config);
	2026	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2027	+ // ONLY check Pinecone if it's enabled
	2028	+ $pinecone_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
3673	2029	if (isset($pinecone_data[$post_id])) {
3674	2030	$is_update = true;
3675	2031	}
3676	2032	} else {
3677		- // Check WordPress DB (same as before since it's shared)
	2033	+ // ONLY check WordPress DB if Pinecone is not enabled
3678	2034	global $wpdb;
3679	2035	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3680	2036	$existing_record = $wpdb->get_row($wpdb->prepare(
3681	2037	"SELECT id FROM $table_name WHERE source_url = %s",
		@@ -3685,54 +2041,54 @@
3685	2041	if ($existing_record) {
3686	2042	$is_update = true;
3687	2043	}
3688	2044	}
3689		-
3690		- // UPDATED 2.5.6: Determine content type based on post_type
3691		- $post_type = $post->post_type;
3692		- $content_type = 'content'; // Default fallback
3693		-
3694		- // Map WordPress post types to content types
3695		- switch ($post_type) {
3696		- case 'post':
3697		- $content_type = 'post';
3698		- break;
3699		- case 'page':
3700		- $content_type = 'page';
3701		- break;
3702		- case 'product':
3703		- $content_type = 'product';
3704		- break;
3705		- default:
3706		- // For custom post types, use the post type name
3707		- $content_type = sanitize_key($post_type);
3708		- break;
3709		- }
3710		-
3711		- // Use the centralized utility function with bot_id and content_type
	2045	+
	2046	+ // Use the centralized utility function for storage
3712	2047	$result = MxChat_Utils::submit_content_to_db(
3713		- $content,
3714		- $source_url,
	2048	+ $content,
	2049	+ $source_url,
3715	2050	$api_key,
3716		- $vector_id,
3717		- $bot_id,
3718		- $content_type
	2051	+ $vector_id
3719	2052	);
3720		-
	2053	+
3721	2054	if (is_wp_error($result)) {
3722		- MxChat_Admin::mxchat_log_debug('storage_error', 'Knowledge storage failed: ' . $result->get_error_message(), array('source_url' => $source_url));
3723	2055	wp_send_json_error('Storage failed: ' . $result->get_error_message());
3724	2056	exit;
3725	2057	}
3726		-
3727		- // Automatically apply role restriction based on tags
3728		- $this->apply_role_restriction_to_post($post_id, $source_url);
3729		-
	2058	+
	2059	+ // ================================
	2060	+ // UPDATE: Only update caches if Pinecone is enabled
	2061	+ // ================================
	2062	+
	2063	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2064	+ // Update vector ID cache for improved fetching
	2065	+ $this->mxchat_update_pinecone_vector_cache($vector_id);
	2066	+
	2067	+ // Update local processed content cache for immediate UI feedback
	2068	+ $pinecone_cache = get_option('mxchat_pinecone_processed_cache', array());
	2069	+ $pinecone_cache[$post_id] = array(
	2070	+ 'db_id' => $vector_id,
	2071	+ 'processed_date' => 'Just now',
	2072	+ 'url' => $source_url,
	2073	+ 'source' => 'pinecone',
	2074	+ 'timestamp' => current_time('timestamp')
	2075	+ );
	2076	+ update_option('mxchat_pinecone_processed_cache', $pinecone_cache);
	2077	+
	2078	+ // Also update the general processed content cache
	2079	+ $processed_cache = get_option('mxchat_processed_content_cache', array());
	2080	+ $processed_cache[$post_id] = array(
	2081	+ 'db_id' => $vector_id,
	2082	+ 'timestamp' => current_time('timestamp'),
	2083	+ 'url' => $source_url,
	2084	+ 'source' => 'pinecone'
	2085	+ );
	2086	+ update_option('mxchat_processed_content_cache', $processed_cache);
	2087	+ }
	2088	+
3730	2089	$operation_type = $is_update ? 'update' : 'new';
3731	2090
3732		- // Count ACF fields for debugging
3733		- $acf_field_count = count($acf_fields);
3734		-
3735	2091	// Success response with minimal data
3736	2092	wp_send_json_success(array(
3737	2093	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
3738	2094	'post_id' => $post_id,
		@@ -3737,94 +2093,27 @@
3737	2093	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
3738	2094	'post_id' => $post_id,
3739	2095	'title' => $post->post_title,
3740	2096	'operation_type' => $operation_type,
3741		- 'vector_id' => $vector_id,
3742		- 'acf_fields_found' => $acf_field_count,
3743		- 'content_preview' => substr($content, 0, 100) . '...',
3744		- 'bot_id' => $bot_id
	2097	+ 'vector_id' => $vector_id, // Include vector ID for debugging
	2098	+ 'cache_updated' => $use_pinecone // Indicate if cache was updated
3745	2099	));
3746	2100	exit;
3747	2101	}
3748	2102
3749		-private function apply_role_restriction_to_post($post_id, $source_url) {
3750		- // Get tag-role mappings
3751		- $mappings = get_option('mxchat_tag_role_mappings', array());
3752		-
3753		- if (empty($mappings)) {
3754		- return; // No mappings, leave as public
3755		- }
3756		-
3757		- // Get all tags for the post
3758		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
3759		-
3760		- if (empty($post_tags)) {
3761		- return; // No tags, leave as public
3762		- }
3763		-
3764		- // Determine the highest role restriction based on tags
3765		- $highest_role = 'public';
3766		- $role_hierarchy = array(
3767		- 'public' => 0,
3768		- 'logged_in' => 1,
3769		- 'subscriber' => 2,
3770		- 'contributor' => 3,
3771		- 'author' => 4,
3772		- 'editor' => 5,
3773		- 'administrator' => 6
3774		- );
3775		-
3776		- foreach ($post_tags as $tag_slug) {
3777		- if (isset($mappings[$tag_slug])) {
3778		- $role = $mappings[$tag_slug];
3779		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
3780		- $highest_role = $role;
3781		- }
3782		- }
3783		- }
3784		-
3785		- // If no restricted tags found, return (leave as public)
3786		- if ($highest_role === 'public') {
3787		- return;
3788		- }
3789		-
3790		- // Update the role restriction in the database
3791		- global $wpdb;
3792		-
3793		- // Check if using Pinecone
3794		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
3795		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3796		-
3797		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3798		- // Update Pinecone role restriction
3799		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
3800		- $vector_id = md5($source_url);
3801		-
3802		- $wpdb->replace(
3803		- $roles_table,
3804		- array(
3805		- 'vector_id' => $vector_id,
3806		- 'role_restriction' => $highest_role,
3807		- 'updated_at' => current_time('mysql')
3808		- ),
3809		- array('%s', '%s', '%s')
3810		- );
3811		- } else {
3812		- // Update WordPress DB
3813		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3814		-
3815		- $wpdb->update(
3816		- $table_name,
3817		- array('role_restriction' => $highest_role),
3818		- array('source_url' => $source_url),
3819		- array('%s'),
3820		- array('%s')
3821		- );
3822		- }
3823		-}
3824	2103
	2104	+
	2105	+ /**
	2106	+ * Updates cache with new vector ID if absent
	2107	+ */
	2108	+ public function mxchat_update_pinecone_vector_cache($vector_id) {
	2109	+ $cached_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2110	+ if (!in_array($vector_id, $cached_ids)) {
	2111	+ $cached_ids[] = $vector_id;
	2112	+ update_option('mxchat_pinecone_vector_ids_cache', $cached_ids);
	2113	+ }
	2114	+ }
3825	2115	public function mxchat_get_public_post_types() {
3826		- // Get all public post types
3827	2116	$post_types = get_post_types(array('public' => true), 'objects');
3828	2117	$post_type_options = array();
3829	2118
3830	2119	foreach ($post_types as $post_type) {
		@@ -3830,47 +2119,41 @@
3830	2119	foreach ($post_types as $post_type) {
3831	2120	$post_type_options[$post_type->name] = $post_type->label;
3832	2121	}
3833	2122
3834		- // Also include common forum/community post types that might not be marked as public
3835		- $additional_types = array(
3836		- 'topic' => 'Forum Topics (bbPress)',
3837		- 'reply' => 'Forum Replies (bbPress)',
3838		- 'forum' => 'Forums (bbPress)',
3839		- 'wpforo_topic' => 'wpForo Topics',
3840		- 'wpforo_post' => 'wpForo Posts'
3841		- );
3842		-
3843		- foreach ($additional_types as $type_name => $type_label) {
3844		- if (post_type_exists($type_name) && !isset($post_type_options[$type_name])) {
3845		- $post_type_options[$type_name] = $type_label;
3846		- }
3847		- }
3848		-
3849	2123	return $post_type_options;
3850	2124	}
3851		-
3852		-/**
3853		- * Retrieves processed content from Pinecone API
3854		- */
3855	2125	public function mxchat_get_pinecone_processed_content($pinecone_options) {
	2126	+ // First check local cache for immediate updates
	2127	+ $cached_data = get_option('mxchat_pinecone_processed_cache', array());
	2128	+
3856	2129	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
3857	2130	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
3858		-
	2131	+
3859	2132	if (empty($api_key) || empty($host)) {
3860		- return array();
	2133	+ // Return only cached data if API credentials are missing
	2134	+ return $cached_data;
3861	2135	}
3862		-
	2136	+
3863	2137	$pinecone_data = array();
3864		-
	2138	+
3865	2139	try {
3866		- // Always get fresh data from Pinecone
3867		- $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
3868		-
3869		- // Method 2: Final fallback - try stats endpoint (if available)
	2140	+ // Method 1: Try to get vectors using cached vector IDs first
	2141	+ $cached_vector_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2142	+
	2143	+ if (!empty($cached_vector_ids)) {
	2144	+ $pinecone_data = $this->mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $cached_vector_ids);
	2145	+ }
	2146	+
	2147	+ // Method 2: If no cached IDs or fetch failed, use scanning approach
3870	2148	if (empty($pinecone_data)) {
	2149	+ $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
	2150	+ }
	2151	+
	2152	+ // Method 3: Final fallback - try stats endpoint (if available)
	2153	+ if (empty($pinecone_data)) {
3871	2154	$stats_url = "https://{$host}/describe_index_stats";
3872		-
	2155	+
3873	2156	$response = wp_remote_post($stats_url, array(
3874	2157	'headers' => array(
3875	2158	'Api-Key' => $api_key,
3876	2159	'Content-Type' => 'application/json'
		@@ -3877,42 +2160,59 @@
3877	2160	),
3878	2161	'body' => json_encode(array()),
3879	2162	'timeout' => 30
3880	2163	));
3881		-
	2164	+
3882	2165	if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3883	2166	$body = wp_remote_retrieve_body($response);
3884	2167	$stats_data = json_decode($body, true);
	2168	+
	2169	+ // Log stats for debugging but don't rely on them for vector listing
	2170	+ //error_log('Pinecone index stats: ' . print_r($stats_data, true));
3885	2171	}
3886	2172	}
3887		-
	2173	+
3888	2174	} catch (Exception $e) {
3889		- // Log error but return fresh data only
	2175	+ //error_log('Pinecone processed content exception: ' . $e->getMessage());
3890	2176	}
3891		-
3892		- return $pinecone_data;
	2177	+
	2178	+ // Merge cached data with Pinecone data
	2179	+ // Cache takes priority for recent updates (within last 5 minutes)
	2180	+ $merged_data = $pinecone_data;
	2181	+
	2182	+ foreach ($cached_data as $post_id => $cache_item) {
	2183	+ $cache_timestamp = $cache_item['timestamp'] ?? 0;
	2184	+ $time_diff = current_time('timestamp') - $cache_timestamp;
	2185	+
	2186	+ // If cache item is recent (less than 5 minutes), prioritize it
	2187	+ if ($time_diff < 300) { // 5 minutes = 300 seconds
	2188	+ $merged_data[$post_id] = $cache_item;
	2189	+ } else {
	2190	+ // If not in Pinecone data and cache is old, keep cache but mark as potentially stale
	2191	+ if (!isset($merged_data[$post_id])) {
	2192	+ $merged_data[$post_id] = $cache_item;
	2193	+ }
	2194	+ }
	2195	+ }
	2196	+
	2197	+ return $merged_data;
3893	2198	}
3894	2199	public function mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $vector_ids) {
3895		- //error_log('=== DEBUG: Starting mxchat_fetch_pinecone_vectors_by_ids ===');
3896		-
3897	2200	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
3898	2201	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
3899		-
	2202	+
3900	2203	if (empty($api_key) || empty($host) || empty($vector_ids)) {
3901		- //error_log('DEBUG: Missing parameters for fetch by IDs');
3902	2204	return array();
3903	2205	}
3904		-
	2206	+
3905	2207	try {
3906	2208	$fetch_url = "https://{$host}/vectors/fetch";
3907		- //error_log('DEBUG: Fetch URL: ' . $fetch_url);
3908		- //error_log('DEBUG: Fetching ' . count($vector_ids) . ' vector IDs');
3909		-
	2209	+
3910	2210	// Pinecone fetch API allows fetching specific vectors by ID
3911	2211	$fetch_data = array(
3912	2212	'ids' => array_values($vector_ids)
3913	2213	);
3914		-
	2214	+
3915	2215	$response = wp_remote_post($fetch_url, array(
3916	2216	'headers' => array(
3917	2217	'Api-Key' => $api_key,
3918	2218	'Content-Type' => 'application/json'
		@@ -3919,45 +2219,39 @@
3919	2219	),
3920	2220	'body' => json_encode($fetch_data),
3921	2221	'timeout' => 30
3922	2222	));
3923		-
	2223	+
3924	2224	if (is_wp_error($response)) {
3925		- //error_log('DEBUG: Fetch by IDs WP error: ' . $response->get_error_message());
	2225	+ //error_log('Pinecone fetch by IDs error: ' . $response->get_error_message());
3926	2226	return array();
3927	2227	}
3928		-
	2228	+
3929	2229	$response_code = wp_remote_retrieve_response_code($response);
3930		- //error_log('DEBUG: Fetch response code: ' . $response_code);
3931		-
3932	2230	if ($response_code !== 200) {
3933		- $error_body = wp_remote_retrieve_body($response);
3934		- //error_log('DEBUG: Fetch failed with body: ' . $error_body);
	2231	+ //error_log('Pinecone fetch by IDs failed with code: ' . $response_code);
3935	2232	return array();
3936	2233	}
3937		-
	2234	+
3938	2235	$body = wp_remote_retrieve_body($response);
3939	2236	$data = json_decode($body, true);
3940	2237
3941		- //error_log('DEBUG: Fetch response structure: ' . print_r(array_keys($data), true));
3942		-
3943	2238	if (!isset($data['vectors'])) {
3944		- //error_log('DEBUG: No vectors key in response');
3945	2239	return array();
3946	2240	}
3947		-
	2241	+
3948	2242	$processed_data = array();
3949		-
	2243	+
3950	2244	foreach ($data['vectors'] as $vector_id => $vector_data) {
3951	2245	$metadata = $vector_data['metadata'] ?? array();
3952	2246	$source_url = $metadata['source_url'] ?? '';
3953		-
	2247	+
3954	2248	if (!empty($source_url)) {
3955	2249	$post_id = url_to_postid($source_url);
3956	2250	if ($post_id) {
3957	2251	$created_at = $metadata['created_at'] ?? '';
3958		- $processed_date = 'Recently';
3959		-
	2252	+ $processed_date = 'Recently'; // Default
	2253	+
3960	2254	if (!empty($created_at)) {
3961	2255	$timestamp = is_numeric($created_at) ? $created_at : strtotime($created_at);
3962	2256	if ($timestamp) {
3963	2257	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
		@@ -3962,9 +2256,9 @@
3962	2256	if ($timestamp) {
3963	2257	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
3964	2258	}
3965	2259	}
3966		-
	2260	+
3967	2261	$processed_data[$post_id] = array(
3968	2262	'db_id' => $vector_id,
3969	2263	'processed_date' => $processed_date,
3970	2264	'url' => $source_url,
		@@ -3973,90 +2267,52 @@
3973	2267	);
3974	2268	}
3975	2269	}
3976	2270	}
3977		-
3978		- //error_log('DEBUG: Processed ' . count($processed_data) . ' vectors from fetch');
	2271	+
3979	2272	return $processed_data;
3980		-
	2273	+
3981	2274	} catch (Exception $e) {
3982		- //error_log('DEBUG: Exception in fetch_pinecone_vectors_by_ids: ' . $e->getMessage());
	2275	+ //error_log('Pinecone fetch by IDs exception: ' . $e->getMessage());
3983	2276	return array();
3984	2277	}
3985	2278	}
3986		-
3987		-/**
3988		- * Get embedding dimensions based on the selected model.
3989		- */
3990		-private function mxchat_get_embedding_dimensions() {
3991		- $options = get_option('mxchat_options', array());
3992		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
3993		-
3994		- $model_dimensions = array(
3995		- 'text-embedding-ada-002' => 1536,
3996		- 'text-embedding-3-small' => 1536,
3997		- 'text-embedding-3-large' => 3072,
3998		- 'voyage-2' => 1024,
3999		- 'voyage-large-2' => 1536,
4000		- 'voyage-3-large' => 2048,
4001		- 'gemini-embedding-001' => 1536,
4002		- );
4003		-
4004		- if (strpos($selected_model, 'voyage-3-large') === 0) {
4005		- $custom_dimensions = $options['voyage_output_dimension'] ?? 2048;
4006		- return intval($custom_dimensions);
4007		- }
4008		-
4009		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4010		- $custom_dimensions = $options['gemini_output_dimension'] ?? 1536;
4011		- return intval($custom_dimensions);
4012		- }
4013		-
4014		- return $model_dimensions[$selected_model] ?? 1536;
4015		-}
4016		-
4017		-/**
4018		- * Scan Pinecone for processed content
4019		- */
4020	2279	public function mxchat_scan_pinecone_for_processed_content($pinecone_options) {
4021	2280	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4022	2281	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
4023		-
	2282	+
4024	2283	if (empty($api_key) || empty($host)) {
4025	2284	return array();
4026	2285	}
4027		-
	2286	+
4028	2287	try {
4029	2288	// Use multiple random vectors to get better coverage
4030	2289	$all_matches = array();
4031	2290	$seen_ids = array();
4032		-
4033		- // Get correct dimensions for the configured embedding model
4034		- $dimensions = $this->mxchat_get_embedding_dimensions();
4035		-
	2291	+
4036	2292	// Try 3 different random vectors to get better coverage
4037	2293	for ($i = 0; $i < 3; $i++) {
4038	2294	$query_url = "https://{$host}/query";
4039		-
	2295	+
4040	2296	// Generate a random unit vector instead of zeros
4041	2297	$random_vector = array();
4042		- for ($j = 0; $j < $dimensions; $j++) {
4043		- $random_vector[] = (rand(-1000, 1000) / 1000.0);
	2298	+ for ($j = 0; $j < 1536; $j++) {
	2299	+ $random_vector[] = (rand(-1000, 1000) / 1000.0); // Random values between -1 and 1
4044	2300	}
4045		-
	2301	+
4046	2302	// Normalize the vector to unit length
4047	2303	$magnitude = sqrt(array_sum(array_map(function($x) { return $x * $x; }, $random_vector)));
4048	2304	if ($magnitude > 0) {
4049	2305	$random_vector = array_map(function($x) use ($magnitude) { return $x / $magnitude; }, $random_vector);
4050	2306	}
4051		-
	2307	+
4052	2308	$query_data = array(
4053	2309	'includeMetadata' => true,
4054	2310	'includeValues' => false,
4055		- 'topK' => 10000,
	2311	+ 'topK' => 10000, // Get many results
4056	2312	'vector' => $random_vector
4057	2313	);
4058		-
	2314	+
4059	2315	$response = wp_remote_post($query_url, array(
4060	2316	'headers' => array(
4061	2317	'Api-Key' => $api_key,
4062	2318	'Content-Type' => 'application/json'
		@@ -4063,22 +2319,21 @@
4063	2319	),
4064	2320	'body' => json_encode($query_data),
4065	2321	'timeout' => 30
4066	2322	));
4067		-
	2323	+
4068	2324	if (is_wp_error($response)) {
4069	2325	continue;
4070	2326	}
4071		-
	2327	+
4072	2328	$response_code = wp_remote_retrieve_response_code($response);
4073		-
4074	2329	if ($response_code !== 200) {
4075	2330	continue;
4076	2331	}
4077		-
	2332	+
4078	2333	$body = wp_remote_retrieve_body($response);
4079	2334	$data = json_decode($body, true);
4080		-
	2335	+
4081	2336	if (isset($data['matches'])) {
4082	2337	foreach ($data['matches'] as $match) {
4083	2338	$match_id = $match['id'] ?? '';
4084	2339	if (!empty($match_id) && !isset($seen_ids[$match_id])) {
		@@ -4087,30 +2342,24 @@
4087	2342	}
4088	2343	}
4089	2344	}
4090	2345	}
4091		-
4092		- // Convert matches to processed data format, grouping by URL to count chunks
	2346	+
	2347	+ // Convert matches to processed data format
4093	2348	$processed_data = array();
4094		- $url_chunk_counts = array();
4095		-
	2349	+ $vector_ids_for_cache = array();
	2350	+
4096	2351	foreach ($all_matches as $match) {
4097	2352	$metadata = $match['metadata'] ?? array();
4098	2353	$source_url = $metadata['source_url'] ?? '';
4099	2354	$match_id = $match['id'] ?? '';
4100		-
	2355	+
4101	2356	if (!empty($source_url) && !empty($match_id)) {
4102	2357	$post_id = url_to_postid($source_url);
4103	2358	if ($post_id) {
4104		- // Count chunks per post_id
4105		- if (!isset($url_chunk_counts[$post_id])) {
4106		- $url_chunk_counts[$post_id] = 0;
4107		- }
4108		- $url_chunk_counts[$post_id]++;
4109		-
4110	2359	$created_at = $metadata['created_at'] ?? '';
4111		- $processed_date = 'Recently';
4112		-
	2360	+ $processed_date = 'Recently'; // Default
	2361	+
4113	2362	if (!empty($created_at)) {
4114	2363	$timestamp = is_numeric($created_at) ? $created_at : strtotime($created_at);
4115	2364	if ($timestamp) {
4116	2365	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
		@@ -4115,312 +2364,242 @@
4115	2364	if ($timestamp) {
4116	2365	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
4117	2366	}
4118	2367	}
4119		-
4120		- // Only store if not already set, or update with newer timestamp
4121		- if (!isset($processed_data[$post_id]) ||
4122		- ($timestamp ?? 0) > ($processed_data[$post_id]['timestamp'] ?? 0)) {
4123		- $processed_data[$post_id] = array(
4124		- 'db_id' => $match_id,
4125		- 'processed_date' => $processed_date,
4126		- 'url' => $source_url,
4127		- 'source' => 'pinecone',
4128		- 'timestamp' => $timestamp ?? current_time('timestamp')
4129		- );
4130		- }
	2368	+
	2369	+ $processed_data[$post_id] = array(
	2370	+ 'db_id' => $match_id,
	2371	+ 'processed_date' => $processed_date,
	2372	+ 'url' => $source_url,
	2373	+ 'source' => 'pinecone',
	2374	+ 'timestamp' => $timestamp ?? current_time('timestamp')
	2375	+ );
	2376	+
	2377	+ $vector_ids_for_cache[] = $match_id;
4131	2378	}
4132	2379	}
4133	2380	}
4134		-
4135		- // Add chunk counts to processed data
4136		- foreach ($url_chunk_counts as $post_id => $chunk_count) {
4137		- if (isset($processed_data[$post_id])) {
4138		- $processed_data[$post_id]['chunk_count'] = $chunk_count;
4139		- }
	2381	+
	2382	+ // Update the vector IDs cache for future use
	2383	+ if (!empty($vector_ids_for_cache)) {
	2384	+ update_option('mxchat_pinecone_vector_ids_cache', $vector_ids_for_cache);
4140	2385	}
4141		-
	2386	+
4142	2387	return $processed_data;
4143		-
	2388	+
4144	2389	} catch (Exception $e) {
	2390	+ //error_log('Pinecone scan exception: ' . $e->getMessage());
4145	2391	return array();
4146	2392	}
4147		-}
4148		-/**
4149		- * Generate embeddings from input text for MXChat with bot support
4150		- */
4151		-private function mxchat_generate_embedding($text, $bot_id = 'default') {
4152		- // Enable detailed logging for debugging
4153		- //error_log('[MXCHAT-EMBED] Starting embedding generation for bot: ' . $bot_id . '. Text length: ' . strlen($text) . ' bytes');
4154		- //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
	2393	+}
4155	2394
4156		- // Get bot-specific options
4157		- $bot_options = $this->get_bot_options($bot_id);
4158		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
4159		-
4160		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4161		- //error_log('[MXCHAT-EMBED] Selected embedding model for bot ' . $bot_id . ': ' . $selected_model);
	2395	+ /**
	2396	+ * Generates embeddings from input text for MXChat
	2397	+ */
	2398	+ private function mxchat_generate_embedding($text) {
	2399	+ // Enable detailed logging for debugging
	2400	+ //error_log('[MXCHAT-EMBED] Starting embedding generation. Text length: ' . strlen($text) . ' bytes');
	2401	+ //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4162	2402
4163		- // Determine provider and endpoint
4164		- if (strpos($selected_model, 'voyage') === 0) {
4165		- $api_key = $options['voyage_api_key'] ?? '';
4166		- $endpoint = 'https://api.voyageai.com/v1/embeddings';
4167		- $provider_name = 'Voyage AI';
4168		- //error_log('[MXCHAT-EMBED] Using Voyage AI API for bot ' . $bot_id);
4169		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
4170		- $api_key = $options['gemini_api_key'] ?? '';
4171		- $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
4172		- $provider_name = 'Google Gemini';
4173		- //error_log('[MXCHAT-EMBED] Using Google Gemini API for bot ' . $bot_id);
4174		- } else {
4175		- $api_key = $options['api_key'] ?? '';
4176		- $endpoint = 'https://api.openai.com/v1/embeddings';
4177		- $provider_name = 'OpenAI';
4178		- //error_log('[MXCHAT-EMBED] Using OpenAI API for bot ' . $bot_id);
4179		- }
	2403	+ $options = get_option('mxchat_options');
	2404	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	2405	+ //error_log('[MXCHAT-EMBED] Selected embedding model: ' . $selected_model);
4180	2406
4181		- //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
	2407	+ // Determine provider and endpoint
	2408	+ if (strpos($selected_model, 'voyage') === 0) {
	2409	+ $api_key = $options['voyage_api_key'] ?? '';
	2410	+ $endpoint = 'https://api.voyageai.com/v1/embeddings';
	2411	+ $provider_name = 'Voyage AI';
	2412	+ //error_log('[MXCHAT-EMBED] Using Voyage AI API');
	2413	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	2414	+ $api_key = $options['gemini_api_key'] ?? '';
	2415	+ $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
	2416	+ $provider_name = 'Google Gemini';
	2417	+ //error_log('[MXCHAT-EMBED] Using Google Gemini API');
	2418	+ } else {
	2419	+ $api_key = $options['api_key'] ?? '';
	2420	+ $endpoint = 'https://api.openai.com/v1/embeddings';
	2421	+ $provider_name = 'OpenAI';
	2422	+ //error_log('[MXCHAT-EMBED] Using OpenAI API');
	2423	+ }
4182	2424
4183		- if (empty($api_key)) {
4184		- $error_message = sprintf('Missing %s API key for bot %s. Please configure your API key in the bot settings.', $provider_name, $bot_id);
4185		- //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
4186		- return $error_message;
4187		- }
	2425	+ //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
4188	2426
4189		- // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
4190		- $estimated_tokens = ceil(str_word_count($text) / 0.75);
4191		- //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
	2427	+ if (empty($api_key)) {
	2428	+ $error_message = sprintf('Missing %s API key. Please configure your API key in the MxChat settings.', $provider_name);
	2429	+ //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
	2430	+ return $error_message;
	2431	+ }
4192	2432
4193		- if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
4194		- //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
4195		- // Consider truncating text here
4196		- }
	2433	+ // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
	2434	+ $estimated_tokens = ceil(str_word_count($text) / 0.75);
	2435	+ //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
4197	2436
4198		- // Prepare request body based on provider
4199		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4200		- // Gemini API format
4201		- $request_body = array(
4202		- 'model' => 'models/' . $selected_model,
4203		- 'content' => array(
4204		- 'parts' => array(
4205		- array('text' => $text)
4206		- )
4207		- )
4208		- );
	2437	+ if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
	2438	+ //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
	2439	+ // Consider truncating text here
	2440	+ }
4209	2441
4210		- // Set output dimensionality to 1536 for consistency with other models
4211		- $request_body['outputDimensionality'] = 1536;
4212		- } else {
4213		- // OpenAI/Voyage API format
4214		- $request_body = array(
4215		- 'model' => $selected_model,
4216		- 'input' => $text
4217		- );
	2442	+ // Prepare request body based on provider
	2443	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2444	+ // Gemini API format
	2445	+ $request_body = array(
	2446	+ 'model' => 'models/' . $selected_model,
	2447	+ 'content' => array(
	2448	+ 'parts' => array(
	2449	+ array('text' => $text)
	2450	+ )
	2451	+ )
	2452	+ );
4218	2453
4219		- // Add output_dimension for voyage-3-large model
4220		- if ($selected_model === 'voyage-3-large') {
4221		- $request_body['output_dimension'] = 2048;
4222		- }
4223		- }
	2454	+ // Set output dimensionality to 1536 for consistency with other models
	2455	+ $request_body['outputDimensionality'] = 1536;
	2456	+ } else {
	2457	+ // OpenAI/Voyage API format
	2458	+ $request_body = array(
	2459	+ 'model' => $selected_model,
	2460	+ 'input' => $text
	2461	+ );
4224	2462
4225		- //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
	2463	+ // Add output_dimension for voyage-3-large model
	2464	+ if ($selected_model === 'voyage-3-large') {
	2465	+ $request_body['output_dimension'] = 2048;
	2466	+ }
	2467	+ }
4226	2468
4227		- // Prepare headers based on provider
4228		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4229		- // Gemini uses API key as query parameter
4230		- $endpoint .= '?key=' . $api_key;
4231		- $headers = array(
4232		- 'Content-Type' => 'application/json'
4233		- );
4234		- } else {
4235		- // OpenAI/Voyage use Bearer token
4236		- $headers = array(
4237		- 'Authorization' => 'Bearer ' . $api_key,
4238		- 'Content-Type' => 'application/json'
4239		- );
4240		- }
	2469	+ //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
4241	2470
4242		- // Make API request
4243		- //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
4244		- $response = wp_remote_post($endpoint, array(
4245		- 'body' => wp_json_encode($request_body),
4246		- 'headers' => $headers,
4247		- 'timeout' => 60 // Increased timeout for large inputs
4248		- ));
	2471	+ // Prepare headers based on provider
	2472	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2473	+ // Gemini uses API key as query parameter
	2474	+ $endpoint .= '?key=' . $api_key;
	2475	+ $headers = array(
	2476	+ 'Content-Type' => 'application/json'
	2477	+ );
	2478	+ } else {
	2479	+ // OpenAI/Voyage use Bearer token
	2480	+ $headers = array(
	2481	+ 'Authorization' => 'Bearer ' . $api_key,
	2482	+ 'Content-Type' => 'application/json'
	2483	+ );
	2484	+ }
4249	2485
4250		- // Handle wp_remote_post errors
4251		- if (is_wp_error($response)) {
4252		- $error_message = $response->get_error_message();
4253		- //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
4254		- return 'Connection error: ' . $error_message;
4255		- }
	2486	+ // Make API request
	2487	+ //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
	2488	+ $response = wp_remote_post($endpoint, array(
	2489	+ 'body' => wp_json_encode($request_body),
	2490	+ 'headers' => $headers,
	2491	+ 'timeout' => 60 // Increased timeout for large inputs
	2492	+ ));
4256	2493
4257		- // Get and check HTTP response code
4258		- $http_code = wp_remote_retrieve_response_code($response);
4259		- //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
	2494	+ // Handle wp_remote_post errors
	2495	+ if (is_wp_error($response)) {
	2496	+ $error_message = $response->get_error_message();
	2497	+ //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
	2498	+ return 'Connection error: ' . $error_message;
	2499	+ }
4260	2500
4261		- if ($http_code !== 200) {
4262		- $error_body = wp_remote_retrieve_body($response);
4263		- //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
	2501	+ // Get and check HTTP response code
	2502	+ $http_code = wp_remote_retrieve_response_code($response);
	2503	+ //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
4264	2504
4265		- // Try to parse error for more details
4266		- $error_json = json_decode($error_body, true);
4267		- if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
4268		- $error_type = $error_json['error']['type'] ?? 'unknown';
4269		- $error_message = $error_json['error']['message'] ?? 'No message';
4270		- //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
4271		- //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
	2505	+ if ($http_code !== 200) {
	2506	+ $error_body = wp_remote_retrieve_body($response);
	2507	+ //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
4272	2508
4273		- // Customize error message for common API errors
4274		- if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
4275		- $error_message = sprintf('Invalid %s API key for bot %s. Please check your API key in the bot settings.', $provider_name, $bot_id);
4276		- } elseif ($error_type === 'authentication_error') {
4277		- $error_message = sprintf('%s authentication failed for bot %s. Please verify your API key in the bot settings.', $provider_name, $bot_id);
4278		- }
	2509	+ // Try to parse error for more details
	2510	+ $error_json = json_decode($error_body, true);
	2511	+ if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
	2512	+ $error_type = $error_json['error']['type'] ?? 'unknown';
	2513	+ $error_message = $error_json['error']['message'] ?? 'No message';
	2514	+ //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
	2515	+ //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
4279	2516
4280		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4281		- return $error_message;
4282		- }
	2517	+ // Customize error message for common API errors
	2518	+ if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
	2519	+ $error_message = sprintf('Invalid %s API key. Please check your API key in the MxChat settings.', $provider_name);
	2520	+ } elseif ($error_type === 'authentication_error') {
	2521	+ $error_message = sprintf('%s authentication failed. Please verify your API key in the MxChat settings.', $provider_name);
	2522	+ }
4283	2523
4284		- $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding for bot %s", $http_code, $bot_id);
4285		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4286		- return $error_message;
4287		- }
	2524	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2525	+ return $error_message;
	2526	+ }
4288	2527
4289		- // Parse response body
4290		- $response_body = wp_remote_retrieve_body($response);
4291		- //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
	2528	+ $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding", $http_code);
	2529	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2530	+ return $error_message;
	2531	+ }
4292	2532
4293		- $response_data = json_decode($response_body, true);
	2533	+ // Parse response body
	2534	+ $response_body = wp_remote_retrieve_body($response);
	2535	+ //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
4294	2536
4295		- if (json_last_error() !== JSON_ERROR_NONE) {
4296		- $error = json_last_error_msg();
4297		- //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
4298		- //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
4299		- return "Failed to parse API response: $error";
4300		- }
	2537	+ $response_data = json_decode($response_body, true);
4301	2538
4302		- // Handle different response formats based on provider
4303		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4304		- // Gemini API response format
4305		- if (isset($response_data['embedding']['values'])) {
4306		- $embedding_dimensions = count($response_data['embedding']['values']);
4307		- //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
	2539	+ if (json_last_error() !== JSON_ERROR_NONE) {
	2540	+ $error = json_last_error_msg();
	2541	+ //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
	2542	+ //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
	2543	+ return "Failed to parse API response: $error";
	2544	+ }
4308	2545
4309		- // Check if embedding dimensions are as expected (should be 1536)
4310		- if ($embedding_dimensions !== 1536) {
4311		- //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
4312		- }
	2546	+ // Handle different response formats based on provider
	2547	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2548	+ // Gemini API response format
	2549	+ if (isset($response_data['embedding']['values'])) {
	2550	+ $embedding_dimensions = count($response_data['embedding']['values']);
	2551	+ //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
4313	2552
4314		- return $response_data['embedding']['values'];
4315		- } else {
4316		- //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
4317		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2553	+ // Check if embedding dimensions are as expected (should be 1536)
	2554	+ if ($embedding_dimensions !== 1536) {
	2555	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
	2556	+ }
4318	2557
4319		- if (isset($response_data['error'])) {
4320		- $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
4321		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4322		- return $error_message;
4323		- }
	2558	+ return $response_data['embedding']['values'];
	2559	+ } else {
	2560	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
	2561	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4324	2562
4325		- $error_message = "Invalid Gemini API response format: No embedding found";
4326		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4327		- return $error_message;
4328		- }
4329		- } else {
4330		- // OpenAI/Voyage API response format
4331		- if (isset($response_data['data'][0]['embedding'])) {
4332		- $embedding_dimensions = count($response_data['data'][0]['embedding']);
4333		- //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
	2563	+ if (isset($response_data['error'])) {
	2564	+ $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
	2565	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2566	+ return $error_message;
	2567	+ }
4334	2568
4335		- // Check if embedding dimensions are as expected
4336		- if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
4337		- ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
4338		- //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
4339		- }
	2569	+ $error_message = "Invalid Gemini API response format: No embedding found";
	2570	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2571	+ return $error_message;
	2572	+ }
	2573	+ } else {
	2574	+ // OpenAI/Voyage API response format
	2575	+ if (isset($response_data['data'][0]['embedding'])) {
	2576	+ $embedding_dimensions = count($response_data['data'][0]['embedding']);
	2577	+ //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
4340	2578
4341		- return $response_data['data'][0]['embedding'];
4342		- } else {
4343		- //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
4344		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2579	+ // Check if embedding dimensions are as expected
	2580	+ if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
	2581	+ ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
	2582	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
	2583	+ }
4345	2584
4346		- if (isset($response_data['error'])) {
4347		- $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
4348		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4349		- return $error_message;
4350		- }
	2585	+ return $response_data['data'][0]['embedding'];
	2586	+ } else {
	2587	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
	2588	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4351	2589
4352		- $error_message = "Invalid API response format: No embedding found";
4353		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4354		- return $error_message;
4355		- }
4356		- }
4357		-}
	2590	+ if (isset($response_data['error'])) {
	2591	+ $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
	2592	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2593	+ return $error_message;
	2594	+ }
4358	2595
4359		-/**
4360		- * Get bot-specific options for multi-bot functionality
4361		- * Falls back to default options if bot_id is 'default' or multi-bot add-on is not active
4362		- */
4363		-private function get_bot_options($bot_id = 'default') {
4364		- //error_log("MXCHAT DEBUG: get_bot_options called for bot: " . $bot_id);
4365		-
4366		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4367		- //error_log("MXCHAT DEBUG: Using default options (no multi-bot or bot is 'default')");
4368		- return array();
4369		- }
4370		-
4371		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
4372		-
4373		- if (!empty($bot_options)) {
4374		- //error_log("MXCHAT DEBUG: Got bot-specific options from filter");
4375		- if (isset($bot_options['similarity_threshold'])) {
4376		- //error_log(" - similarity_threshold: " . $bot_options['similarity_threshold']);
4377		- }
4378		- }
4379		-
4380		- return is_array($bot_options) ? $bot_options : array();
4381		-}
4382		-
4383		-/**
4384		- * Get bot-specific Pinecone configuration
4385		- * Used in the knowledge retrieval functions
4386		- */
4387		-// Also add debugging to your get_bot_pinecone_config function
4388		-private function get_bot_pinecone_config($bot_id = 'default') {
4389		- //error_log("MXCHAT DEBUG: get_bot_pinecone_config called for bot: " . $bot_id);
4390		-
4391		- // If default bot or multi-bot add-on not active, use default Pinecone config
4392		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4393		- //error_log("MXCHAT DEBUG: Using default Pinecone config (no multi-bot or bot is 'default')");
4394		- $addon_options = get_option('mxchat_pinecone_addon_options', array());
4395		- $config = array(
4396		- 'use_pinecone' => (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1'),
4397		- 'api_key' => $addon_options['mxchat_pinecone_api_key'] ?? '',
4398		- 'host' => $addon_options['mxchat_pinecone_host'] ?? '',
4399		- 'namespace' => $addon_options['mxchat_pinecone_namespace'] ?? ''
4400		- );
4401		- //error_log("MXCHAT DEBUG: Default config - use_pinecone: " . ($config['use_pinecone'] ? 'true' : 'false'));
4402		- return $config;
4403		- }
4404		-
4405		- //error_log("MXCHAT DEBUG: Calling filter 'mxchat_get_bot_pinecone_config' for bot: " . $bot_id);
4406		-
4407		- // Hook for multi-bot add-on to provide bot-specific Pinecone config
4408		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
4409		-
4410		- if (!empty($bot_pinecone_config)) {
4411		- //error_log("MXCHAT DEBUG: Got bot-specific config from filter");
4412		- //error_log(" - use_pinecone: " . (isset($bot_pinecone_config['use_pinecone']) ? ($bot_pinecone_config['use_pinecone'] ? 'true' : 'false') : 'not set'));
4413		- //error_log(" - host: " . ($bot_pinecone_config['host'] ?? 'not set'));
4414		- //error_log(" - namespace: " . ($bot_pinecone_config['namespace'] ?? 'not set'));
4415		- } else {
4416		- //error_log("MXCHAT DEBUG: Filter returned empty config!");
4417		- }
4418		-
4419		- return is_array($bot_pinecone_config) ? $bot_pinecone_config : array();
4420		-}
4421		-
4422		-
	2596	+ $error_message = "Invalid API response format: No embedding found";
	2597	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2598	+ return $error_message;
	2599	+ }
	2600	+ }
	2601	+ }
4423	2602	public function mxchat_ajax_dismiss_completed_status() {
4424	2603	try {
4425	2604	// Verify the request
4426	2605	check_ajax_referer('mxchat_status_nonce', 'nonce');
		@@ -4539,10 +2718,10 @@
4539	2718
4540	2719	$html .= '<p><strong>' . esc_html__('Status:', 'mxchat') . '</strong> ' . esc_html(ucfirst($status['status'])) . '</p>';
4541	2720	$html .= '<p><strong>' . esc_html__('Last update:', 'mxchat') . '</strong> ' . esc_html($status['last_update']) . '</p>';
4542	2721
4543		- // Add completion summary if available AND it's an array
4544		- if (isset($status['completion_summary']) && is_array($status['completion_summary']) && !empty($status['completion_summary'])) {
	2722	+ // Add completion summary if available
	2723	+ if (isset($status['completion_summary'])) {
4545	2724	$summary = $status['completion_summary'];
4546	2725	$html .= '<div class="mxchat-completion-summary">';
4547	2726	$html .= '<h5>' . esc_html__('Processing Summary', 'mxchat') . '</h5>';
4548	2727	$html .= '<p><strong>' . esc_html__('Total Pages:', 'mxchat') . '</strong> ' . esc_html($summary['total_pages']) . '</p>';
		@@ -4551,10 +2730,10 @@
4551	2730	$html .= '<p><strong>' . esc_html__('Completed:', 'mxchat') . '</strong> ' . esc_html($summary['completion_time']) . '</p>';
4552	2731	$html .= '</div>';
4553	2732	}
4554	2733
4555		- // Add failed pages list if any AND it's an array
4556		- if (isset($status['failed_pages_list']) && is_array($status['failed_pages_list']) && !empty($status['failed_pages_list'])) {
	2734	+ // Add failed pages list if any
	2735	+ if (!empty($status['failed_pages_list'])) {
4557	2736	$html .= $this->mxchat_render_failed_pages_list($status['failed_pages_list']);
4558	2737	}
4559	2738
4560	2739	// Add error message if any
		@@ -4568,8 +2747,9 @@
4568	2747	$html .= '</div>'; // End card
4569	2748
4570	2749	return $html;
4571	2750	}
	2751	+
4572	2752	/**
4573	2753	* Render sitemap status card HTML
4574	2754	*/
4575	2755	private function mxchat_render_sitemap_status_card($status, $sitemap_url) {
		@@ -4625,10 +2805,10 @@
4625	2805
4626	2806	$html .= '<p><strong>' . esc_html__('Status:', 'mxchat') . '</strong> ' . esc_html(ucfirst($status['status'])) . '</p>';
4627	2807	$html .= '<p><strong>' . esc_html__('Last update:', 'mxchat') . '</strong> ' . esc_html($status['last_update']) . '</p>';
4628	2808
4629		- // Add completion summary if available AND it's an array
4630		- if (isset($status['completion_summary']) && is_array($status['completion_summary']) && !empty($status['completion_summary'])) {
	2809	+ // Add completion summary if available
	2810	+ if (isset($status['completion_summary'])) {
4631	2811	$summary = $status['completion_summary'];
4632	2812	$html .= '<div class="mxchat-completion-summary">';
4633	2813	$html .= '<h5>' . esc_html__('Processing Summary', 'mxchat') . '</h5>';
4634	2814	$html .= '<p><strong>' . esc_html__('Total URLs:', 'mxchat') . '</strong> ' . esc_html($summary['total_urls']) . '</p>';
		@@ -4657,16 +2837,13 @@
4657	2837	$html .= '</div>'; // End card
4658	2838
4659	2839	return $html;
4660	2840	}
4661		-
4662		-
4663	2841	/**
4664	2842	* Render failed pages list
4665	2843	*/
4666	2844	private function mxchat_render_failed_pages_list($failed_pages_list) {
4667		- // Validate that $failed_pages_list is an array and not empty
4668		- if (!is_array($failed_pages_list) || empty($failed_pages_list)) {
	2845	+ if (empty($failed_pages_list)) {
4669	2846	return '';
4670	2847	}
4671	2848
4672	2849	$html = '<div class="mxchat-error-notice">';
		@@ -4691,13 +2868,8 @@
4691	2868	return ($b['time'] ?? 0) - ($a['time'] ?? 0);
4692	2869	});
4693	2870
4694	2871	foreach ($sorted_failed_pages as $item) {
4695		- // Ensure $item is an array before accessing its elements
4696		- if (!is_array($item)) {
4697		- continue;
4698		- }
4699		-
4700	2872	$time_ago = isset($item['time']) ? human_time_diff($item['time'], current_time('timestamp')) . ' ' . esc_html__('ago', 'mxchat') : 'Unknown';
4701	2873	$html .= '<tr>';
4702	2874	$html .= '<td>' . esc_html__('Page', 'mxchat') . ' ' . esc_html($item['page'] ?? 'Unknown') . '</td>';
4703	2875	$html .= '<td style="word-break: break-word;">' . esc_html($item['error'] ?? 'Unknown error') . '</td>';
		@@ -4715,10 +2887,9 @@
4715	2887	/**
4716	2888	* Render failed URLs list
4717	2889	*/
4718	2890	private function mxchat_render_failed_urls_list($failed_urls_list) {
4719		- // Validate that $failed_urls_list is an array and not empty
4720		- if (!is_array($failed_urls_list) || empty($failed_urls_list)) {
	2891	+ if (empty($failed_urls_list)) {
4721	2892	return '';
4722	2893	}
4723	2894
4724	2895	$html = '<div class="mxchat-failed-urls-container">';
		@@ -4745,13 +2916,8 @@
4745	2916	// Show up to 50 failed URLs
4746	2917	$display_urls = array_slice($sorted_failed_urls, 0, 50);
4747	2918
4748	2919	foreach ($display_urls as $item) {
4749		- // Ensure $item is an array before accessing its elements
4750		- if (!is_array($item)) {
4751		- continue;
4752		- }
4753		-
4754	2920	$url = $item['url'] ?? '';
4755	2921	$time_ago = isset($item['time']) ? human_time_diff($item['time'], current_time('timestamp')) . ' ' . esc_html__('ago', 'mxchat') : 'Unknown';
4756	2922
4757	2923	// Truncate URL for display
		@@ -4783,375 +2949,18 @@
4783	2949
4784	2950	return $html;
4785	2951	}
4786	2952
4787		-/**
4788		- * Get all ACF fields for a specific post, excluding any fields the user has disabled
4789		- */
4790		-public function mxchat_get_acf_fields_for_post($post_id) {
4791		- if (!function_exists('get_fields')) {
4792		- return array();
4793		- }
4794		-
4795		- $fields = get_fields($post_id);
4796		- if (!$fields || !is_array($fields)) {
4797		- return array();
4798		- }
4799		-
4800		- // Get excluded fields from settings
4801		- $excluded_fields = get_option('mxchat_acf_excluded_fields', array());
4802		- if (!empty($excluded_fields) && is_array($excluded_fields)) {
4803		- foreach ($excluded_fields as $excluded_field) {
4804		- if (isset($fields[$excluded_field])) {
4805		- unset($fields[$excluded_field]);
4806		- }
4807		- }
4808		- }
4809		-
4810		- return $fields;
4811		-}
4812		-
4813		-/**
4814		- * Get all registered ACF field groups and their fields for the settings UI
4815		- */
4816		-public function mxchat_get_all_acf_fields() {
4817		- if (!function_exists('acf_get_field_groups') || !function_exists('acf_get_fields')) {
4818		- return array();
4819		- }
4820		-
4821		- $all_fields = array();
4822		- $field_groups = acf_get_field_groups();
4823		-
4824		- if (!empty($field_groups)) {
4825		- foreach ($field_groups as $group) {
4826		- $group_fields = acf_get_fields($group['key']);
4827		- if (!empty($group_fields)) {
4828		- $all_fields[$group['title']] = array();
4829		- foreach ($group_fields as $field) {
4830		- $all_fields[$group['title']][] = array(
4831		- 'name' => $field['name'],
4832		- 'label' => $field['label'],
4833		- 'type' => $field['type']
4834		- );
4835		- }
4836		- }
4837		- }
4838		- }
4839		-
4840		- return $all_fields;
4841		-}
4842		-
4843		-/**
4844		- * Get whitelisted custom post meta for a given post
4845		- * This allows non-ACF meta fields (like OptionTree, theme meta boxes) to be included in embeddings
4846		- */
4847		-public function mxchat_get_whitelisted_post_meta($post_id) {
4848		- $whitelist = get_option('mxchat_custom_meta_whitelist', '');
4849		-
4850		- if (empty($whitelist)) {
4851		- return array();
4852		- }
4853		-
4854		- // Parse the whitelist - one meta key per line
4855		- $meta_keys = array_filter(array_map('trim', explode("\n", $whitelist)));
4856		-
4857		- if (empty($meta_keys)) {
4858		- return array();
4859		- }
4860		-
4861		- $result = array();
4862		-
4863		- foreach ($meta_keys as $key) {
4864		- // Skip empty keys
4865		- if (empty($key)) {
4866		- continue;
4867		- }
4868		-
4869		- $value = get_post_meta($post_id, $key, true);
4870		-
4871		- // Only include non-empty string values
4872		- if (!empty($value) && is_string($value)) {
4873		- $result[$key] = $value;
4874		- } elseif (!empty($value) && is_array($value)) {
4875		- // Handle array values by joining them
4876		- $flat_value = $this->mxchat_flatten_meta_array($value);
4877		- if (!empty($flat_value)) {
4878		- $result[$key] = $flat_value;
4879		- }
4880		- }
4881		- }
4882		-
4883		- return $result;
4884		-}
4885		-
4886		-/**
4887		- * Flatten array meta values into a readable string
4888		- */
4889		-private function mxchat_flatten_meta_array($array, $depth = 0) {
4890		- if ($depth > 3) {
4891		- return ''; // Prevent infinite recursion
4892		- }
4893		-
4894		- $parts = array();
4895		-
4896		- foreach ($array as $key => $value) {
4897		- if (is_string($value) && !empty($value)) {
4898		- $parts[] = $value;
4899		- } elseif (is_array($value)) {
4900		- $nested = $this->mxchat_flatten_meta_array($value, $depth + 1);
4901		- if (!empty($nested)) {
4902		- $parts[] = $nested;
4903		- }
4904		- }
4905		- }
4906		-
4907		- return implode(', ', $parts);
4908		-}
4909		-
4910		-/**
4911		- * Format ACF field values for content extraction
4912		- */
4913		-public function mxchat_format_acf_field_value($value, $field_name = '', $post_id = 0) {
4914		- if (empty($value)) {
4915		- return '';
4916		- }
4917		-
4918		- // Handle WP_Post objects first (THIS IS THE KEY FIX)
4919		- if ($value instanceof WP_Post) {
4920		- return $value->post_title ?: '';
4921		- }
4922		-
4923		- // Handle other WP objects
4924		- if (is_object($value)) {
4925		- if (isset($value->post_title)) {
4926		- return $value->post_title;
4927		- } elseif (isset($value->display_name)) {
4928		- return $value->display_name;
4929		- } elseif (isset($value->name)) {
4930		- return $value->name;
4931		- } elseif (method_exists($value, '__toString')) {
4932		- try {
4933		- return (string) $value;
4934		- } catch (Exception $e) {
4935		- return '';
4936		- }
4937		- }
4938		- // For any other objects, return empty string
4939		- return '';
4940		- }
4941		-
4942		- // Handle different ACF field types
4943		- if (is_array($value)) {
4944		- // Check if it's an image/file field
4945		- if (isset($value['url'])) {
4946		- // Image field - return alt text, title, or caption
4947		- if (!empty($value['alt'])) {
4948		- return $value['alt'];
4949		- } elseif (!empty($value['title'])) {
4950		- return $value['title'];
4951		- } elseif (!empty($value['caption'])) {
4952		- return $value['caption'];
4953		- } else {
4954		- return ''; // Don't include just the URL
4955		- }
4956		- }
4957		-
4958		- // Check if it's a post object or relationship field
4959		- if (isset($value['post_title'])) {
4960		- return $value['post_title'];
4961		- }
4962		-
4963		- // Check if it's a user field
4964		- if (isset($value['display_name'])) {
4965		- return $value['display_name'];
4966		- }
4967		-
4968		- // Check if it's a taxonomy term
4969		- if (isset($value['name']) && isset($value['taxonomy'])) {
4970		- return $value['name'];
4971		- }
4972		-
4973		- // Check if it's a select field with label
4974		- if (isset($value['label'])) {
4975		- return $value['label'];
4976		- }
4977		-
4978		- // Check for repeater field or flexible content
4979		- if (is_numeric(key($value))) {
4980		- $sub_values = array();
4981		- foreach ($value as $sub_item) {
4982		- if (is_array($sub_item)) {
4983		- // For repeater/flexible content, extract text values
4984		- $sub_text = $this->mxchat_extract_text_from_acf_array($sub_item);
4985		- if (!empty($sub_text)) {
4986		- $sub_values[] = $sub_text;
4987		- }
4988		- } elseif ($sub_item instanceof WP_Post) {
4989		- // Handle WP_Post objects in arrays
4990		- $sub_values[] = $sub_item->post_title ?: '';
4991		- } else {
4992		- $sub_values[] = (string) $sub_item;
4993		- }
4994		- }
4995		- return implode(', ', array_filter($sub_values));
4996		- }
4997		-
4998		- // For other arrays, try to extract meaningful text
4999		- $text_values = array();
5000		- foreach ($value as $key => $val) {
5001		- if (is_string($val) && !empty(trim($val))) {
5002		- $text_values[] = trim($val);
5003		- } elseif ($val instanceof WP_Post) {
5004		- // Handle WP_Post objects in associative arrays
5005		- $text_values[] = $val->post_title ?: '';
5006		- } elseif (is_array($val) && isset($val['post_title'])) {
5007		- $text_values[] = $val['post_title'];
5008		- } elseif (is_array($val) && isset($val['name'])) {
5009		- $text_values[] = $val['name'];
5010		- }
5011		- }
5012		-
5013		- return implode(', ', array_filter($text_values));
5014		- }
5015		-
5016		- // Handle boolean values
5017		- if (is_bool($value)) {
5018		- return $value ? 'Yes' : 'No';
5019		- }
5020		-
5021		- // Handle numeric values
5022		- if (is_numeric($value)) {
5023		- return (string) $value;
5024		- }
5025		-
5026		- // Handle string values
5027		- if (is_string($value)) {
5028		- return trim($value);
5029		- }
5030		-
5031		- // For anything else that we can't handle, return empty string
5032		- // This prevents the "Object could not be converted to string" error
5033		- return '';
5034		-}
5035		-
5036		-/**
5037		- * Extract text from complex ACF array structures
5038		- */
5039		-private function mxchat_extract_text_from_acf_array($array) {
5040		- if (!is_array($array)) {
5041		- return '';
5042		- }
5043		-
5044		- $text_parts = array();
5045		-
5046		- foreach ($array as $key => $value) {
5047		- if (is_string($value) && !empty(trim($value))) {
5048		- // Skip keys that are likely to be IDs or technical values
5049		- if (!is_numeric($value) || strlen($value) > 10) {
5050		- $text_parts[] = trim($value);
5051		- }
5052		- } elseif ($value instanceof WP_Post) {
5053		- // Handle WP_Post objects
5054		- $text_parts[] = $value->post_title ?: '';
5055		- } elseif (is_array($value)) {
5056		- if (isset($value['post_title'])) {
5057		- $text_parts[] = $value['post_title'];
5058		- } elseif (isset($value['name'])) {
5059		- $text_parts[] = $value['name'];
5060		- } elseif (isset($value['label'])) {
5061		- $text_parts[] = $value['label'];
5062		- }
5063		- } elseif (is_object($value)) {
5064		- // Handle other objects safely
5065		- if (isset($value->post_title)) {
5066		- $text_parts[] = $value->post_title;
5067		- } elseif (isset($value->name)) {
5068		- $text_parts[] = $value->name;
5069		- } elseif (isset($value->display_name)) {
5070		- $text_parts[] = $value->display_name;
5071		- }
5072		- }
5073		- }
5074		-
5075		- return implode(', ', array_filter($text_parts));
5076		-}
5077		-
5078		-/**
5079		- * Handle ACF save - fires after ACF fields are saved
5080		- * This ensures ACF field data is available when syncing to knowledge base
5081		- */
5082		-public function mxchat_handle_acf_save($post_id) {
5083		- // Skip if not a valid post
5084		- if (!$post_id || $post_id === 'options') {
5085		- return;
5086		- }
5087		-
5088		- // Skip autosaves and revisions
	2953	+public function mxchat_handle_post_update($post_id, $post, $update) {
	2954	+ // Basic validation checks
5089	2955	if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5090	2956	return;
5091	2957	}
5092		-
5093		- $post = get_post($post_id);
5094		- if (!$post) {
5095		- return;
5096		- }
5097		-
5098		- $post_type = $post->post_type;
5099		-
5100		- // Check if sync is enabled for this post type
5101		- $should_sync = false;
5102		-
5103		- if ($post_type === 'post' && get_option('mxchat_auto_sync_posts') === '1') {
5104		- $should_sync = true;
5105		- } else if ($post_type === 'page' && get_option('mxchat_auto_sync_pages') === '1') {
5106		- $should_sync = true;
5107		- } else if ($post_type === 'product' && class_exists('WooCommerce')) {
5108		- // WooCommerce products - check if WooCommerce integration is enabled
5109		- $options = get_option('mxchat_options', array());
5110		- if (isset($options['enable_woocommerce_integration']) &&
5111		- ($options['enable_woocommerce_integration'] === '1' || $options['enable_woocommerce_integration'] === 'on')) {
5112		- $should_sync = true;
5113		- }
5114		- } else {
5115		- // Check custom post types
5116		- $option_name = 'mxchat_auto_sync_' . $post_type;
5117		- if (get_option($option_name) === '1') {
5118		- $should_sync = true;
5119		- }
5120		- }
5121		-
5122		- if (!$should_sync) {
5123		- return;
5124		- }
5125		-
5126		- // Only process published posts
	2958	+
	2959	+ // Only process published content
5127	2960	if ($post->post_status !== 'publish') {
5128	2961	return;
5129	2962	}
5130		-
5131		- // Check if this post has any ACF fields - if not, no need to re-sync
5132		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
5133		- if (empty($acf_fields)) {
5134		- return;
5135		- }
5136		-
5137		- // Use a transient to prevent duplicate processing (post_updated may have already run)
5138		- $transient_key = 'mxchat_acf_synced_' . $post_id;
5139		- if (get_transient($transient_key)) {
5140		- return;
5141		- }
5142		- set_transient($transient_key, true, 60); // Prevent re-processing for 60 seconds
5143		-
5144		- // Re-run the sync with ACF data now available
5145		- // We pass $update=true since this is effectively an update with ACF data
5146		- $this->mxchat_handle_post_update($post_id, $post, true);
5147		-}
5148		-
5149		-public function mxchat_handle_post_update($post_id, $post, $update) {
5150		- // Basic validation checks
5151		- if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5152		- return;
5153		- }
5154	2963
5155	2964	$post_type = $post->post_type;
5156	2965
5157	2966	// Check if sync is enabled for this post type
		@@ -5173,250 +2982,79 @@
5173	2982	if (!$should_sync) {
5174	2983	return;
5175	2984	}
5176	2985
5177		- // Check if we have stored the previous status and URL in our transients
5178		- $previous_status_key = 'mxchat_prev_status_' . $post_id;
5179		- $previous_status = get_transient($previous_status_key);
	2986	+ // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
	2987	+ $title = get_the_title($post_id);
	2988	+ $content = get_post_field('post_content', $post_id);
5180	2989
5181		- $previous_url_key = 'mxchat_prev_url_' . $post_id;
5182		- $previous_url = get_transient($previous_url_key);
	2990	+ // Apply WordPress content filters to get properly formatted content
	2991	+ $content = apply_filters('the_content', $content);
5183	2992
5184		- // If the post was previously published but is now not published, remove from knowledge base
5185		- if ($previous_status === 'publish' && $post->post_status !== 'publish') {
5186		- // Use the stored URL from when it was published, or fall back to current permalink
5187		- $source_url = $previous_url ?: get_permalink($post_id);
	2993	+ // Strip tags but preserve structure
	2994	+ $content = wp_strip_all_tags($content);
	2995	+
	2996	+ // Combine title and content
	2997	+ $final_content = $title . "\n\n" . $content;
	2998	+
	2999	+ // For custom post types like job_listing, include additional fields
	3000	+ if ($post_type === 'job_listing') {
	3001	+ // Add job-specific meta if available
	3002	+ $job_location = get_post_meta($post_id, '_job_location', true);
	3003	+ if (!empty($job_location)) {
	3004	+ $final_content .= "\n\nLocation: " . $job_location;
	3005	+ }
5188	3006
5189		- if ($source_url) {
5190		- // Check if Pinecone is enabled
5191		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
5192		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5193		-
5194		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
5195		- // Delete from Pinecone
5196		- $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
5197		- } else {
5198		- // Delete from WordPress DB
5199		- global $wpdb;
5200		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
5201		-
5202		- $result = $wpdb->delete(
5203		- $table_name,
5204		- array('source_url' => $source_url),
5205		- array('%s')
5206		- );
	3007	+ // Get job type terms
	3008	+ $job_types = get_the_terms($post_id, 'job_listing_type');
	3009	+ if (!empty($job_types) && !is_wp_error($job_types)) {
	3010	+ $types = array();
	3011	+ foreach ($job_types as $type) {
	3012	+ $types[] = $type->name;
5207	3013	}
	3014	+ $final_content .= "\n\nJob Type: " . implode(', ', $types);
5208	3015	}
5209	3016
5210		- // Clean up the transients and exit early
5211		- delete_transient($previous_status_key);
5212		- delete_transient($previous_url_key);
5213		- return;
	3017	+ // Get company name if available
	3018	+ $company_name = get_post_meta($post_id, '_company_name', true);
	3019	+ if (!empty($company_name)) {
	3020	+ $final_content .= "\n\nCompany: " . $company_name;
	3021	+ }
5214	3022	}
5215	3023
5216		- // Store the current status for next time (if this is an update)
5217		- if ($update) {
5218		- set_transient($previous_status_key, $post->post_status, DAY_IN_SECONDS);
5219		-
5220		- // If the post is currently published, also store its URL
5221		- if ($post->post_status === 'publish') {
5222		- $current_url = get_permalink($post_id);
5223		- set_transient($previous_url_key, $current_url, DAY_IN_SECONDS);
5224		- }
	3024	+ // Get the source URL
	3025	+ $source_url = get_permalink($post_id);
	3026	+
	3027	+ // Get API key with proper model detection
	3028	+ $options = get_option('mxchat_options');
	3029	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	3030	+
	3031	+ if (strpos($selected_model, 'voyage') === 0) {
	3032	+ $api_key = $options['voyage_api_key'] ?? '';
	3033	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	3034	+ $api_key = $options['gemini_api_key'] ?? '';
	3035	+ } else {
	3036	+ $api_key = $options['api_key'] ?? '';
5225	3037	}
5226	3038
5227		- // Only process currently published content for adding/updating
5228		- if ($post->post_status === 'publish') {
5229		- // Get the source URL
5230		- $source_url = get_permalink($post_id);
5231		-
5232		- // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
5233		- $title = get_the_title($post_id);
5234		- $content = get_post_field('post_content', $post_id);
5235		- $excerpt = get_post_field('post_excerpt', $post_id);
5236		-
5237		- // Remove shortcode tags but preserve content inside them
5238		- $content = $this->strip_shortcode_tags_preserve_content($content);
5239		- $excerpt = $this->strip_shortcode_tags_preserve_content($excerpt);
5240		-
5241		- // Strip tags but preserve structure (don't use 'the_content' filter as it may re-add shortcodes)
5242		- $content = wp_strip_all_tags($content);
5243		-
5244		- // Combine title, short description (if exists), and content
5245		- $final_content = $title . "\n\n";
5246		-
5247		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
5248		- if (!empty($excerpt)) {
5249		- $final_content .= "Short Description: " . wp_strip_all_tags($excerpt) . "\n\n";
5250		- }
5251		-
5252		- $final_content .= $content;
5253		-
5254		- // For WooCommerce products, include pricing and product details
5255		- if ($post_type === 'product' && class_exists('WooCommerce')) {
5256		- $product = wc_get_product($post_id);
5257		-
5258		- if ($product) {
5259		- // Get pricing information
5260		- $regular_price = $product->get_regular_price();
5261		- $sale_price = $product->get_sale_price();
5262		- $price = $product->get_price();
5263		- $sku = $product->get_sku();
5264		-
5265		- // Get currency symbol
5266		- $currency_symbol = get_woocommerce_currency_symbol();
5267		-
5268		- // Add pricing information
5269		- $final_content .= "\n";
5270		- if (!empty($regular_price)) {
5271		- $final_content .= "Price: " . $currency_symbol . $regular_price . "\n";
5272		- } elseif (!empty($price)) {
5273		- $final_content .= "Price: " . $currency_symbol . $price . "\n";
5274		- }
5275		-
5276		- if (!empty($sale_price) && $sale_price !== $regular_price) {
5277		- $final_content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
5278		- }
5279		-
5280		- // Handle variable products - show price range
5281		- if ($product->is_type('variable')) {
5282		- $min_price = $product->get_variation_price('min');
5283		- $max_price = $product->get_variation_price('max');
5284		- if ($min_price !== $max_price) {
5285		- $final_content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
5286		- }
5287		- }
5288		-
5289		- if (!empty($sku)) {
5290		- $final_content .= "SKU: " . $sku . "\n";
5291		- }
5292		-
5293		- // Get product categories
5294		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
5295		- if (!empty($categories) && !is_wp_error($categories)) {
5296		- $final_content .= "Categories: " . implode(', ', $categories) . "\n";
5297		- }
5298		- }
5299		- }
5300		-
5301		- // For custom post types like job_listing, include additional fields
5302		- if ($post_type === 'job_listing') {
5303		- // Add job-specific meta if available
5304		- $job_location = get_post_meta($post_id, '_job_location', true);
5305		- if (!empty($job_location)) {
5306		- $final_content .= "\n\nLocation: " . $job_location;
5307		- }
5308		-
5309		- // Get job type terms
5310		- $job_types = get_the_terms($post_id, 'job_listing_type');
5311		- if (!empty($job_types) && !is_wp_error($job_types)) {
5312		- $types = array();
5313		- foreach ($job_types as $type) {
5314		- $types[] = $type->name;
5315		- }
5316		- $final_content .= "\n\nJob Type: " . implode(', ', $types);
5317		- }
5318		-
5319		- // Get company name if available
5320		- $company_name = get_post_meta($post_id, '_company_name', true);
5321		- if (!empty($company_name)) {
5322		- $final_content .= "\n\nCompany: " . $company_name;
5323		- }
5324		- }
5325		-
5326		- // ADD ACF FIELDS SUPPORT (matches ajax_mxchat_process_selected_content behavior)
5327		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
5328		- if (!empty($acf_fields)) {
5329		- $acf_content_parts = array();
5330		-
5331		- foreach ($acf_fields as $field_name => $field_value) {
5332		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
5333		- if (!empty($formatted_value)) {
5334		- // Convert field name to readable label
5335		- $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name));
5336		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
5337		- }
5338		- }
5339		-
5340		- if (!empty($acf_content_parts)) {
5341		- $final_content .= "\n\n" . implode("\n", $acf_content_parts);
5342		- }
5343		- }
5344		-
5345		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
5346		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
5347		- if (!empty($custom_meta)) {
5348		- $meta_content_parts = array();
5349		-
5350		- foreach ($custom_meta as $meta_key => $meta_value) {
5351		- // Convert meta key to readable label
5352		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
5353		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
5354		- }
5355		-
5356		- if (!empty($meta_content_parts)) {
5357		- $final_content .= "\n\n" . implode("\n", $meta_content_parts);
5358		- }
5359		- }
5360		-
5361		- // Get API key with proper model detection
5362		- $options = get_option('mxchat_options');
5363		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
5364		-
5365		- if (strpos($selected_model, 'voyage') === 0) {
5366		- $api_key = $options['voyage_api_key'] ?? '';
5367		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
5368		- $api_key = $options['gemini_api_key'] ?? '';
5369		- } else {
5370		- $api_key = $options['api_key'] ?? '';
5371		- }
5372		-
5373		- if (empty($api_key)) {
5374		- return;
5375		- }
5376		-
5377		- // Use the centralized utility function for storage
5378		- $result = MxChat_Utils::submit_content_to_db(
5379		- $final_content,
5380		- $source_url,
5381		- $api_key,
5382		- md5($source_url) // Vector ID for Pinecone
5383		- );
5384		-
5385		- // After successful storage, apply role restriction based on tags
5386		- if (!is_wp_error($result)) {
5387		- $this->apply_role_restriction_to_post($post_id, $source_url);
5388		- }
	3039	+ if (empty($api_key)) {
	3040	+ error_log('MxChat Auto-sync: No API key configured for embedding model');
	3041	+ return;
5389	3042	}
5390	3043
5391		- // Clean up the stored previous status if not used above
5392		- if ($previous_status !== 'publish' || $post->post_status === 'publish') {
5393		- delete_transient($previous_status_key);
5394		- delete_transient($previous_url_key);
	3044	+ // Use the centralized utility function for storage
	3045	+ $result = MxChat_Utils::submit_content_to_db(
	3046	+ $final_content,
	3047	+ $source_url,
	3048	+ $api_key,
	3049	+ md5($source_url) // Vector ID for Pinecone
	3050	+ );
	3051	+
	3052	+ if (is_wp_error($result)) {
	3053	+ error_log('MxChat Auto-sync failed for post ' . $post_id . ': ' . $result->get_error_message());
5395	3054	}
5396	3055	}
5397	3056
5398		-/**
5399		- * Store the post status and URL before update to detect status transitions
5400		- * This runs before the post is actually updated in the database
5401		- */
5402		-public function mxchat_store_pre_update_status($post_id, $data) {
5403		- // Get the current post from database (before update)
5404		- $current_post = get_post($post_id);
5405		-
5406		- if ($current_post) {
5407		- // Store the current status temporarily
5408		- $status_key = 'mxchat_prev_status_' . $post_id;
5409		- set_transient($status_key, $current_post->post_status, HOUR_IN_SECONDS);
5410		-
5411		- // If the post is currently published, also store its URL
5412		- if ($current_post->post_status === 'publish') {
5413		- $url_key = 'mxchat_prev_url_' . $post_id;
5414		- $current_url = get_permalink($post_id);
5415		- set_transient($url_key, $current_url, HOUR_IN_SECONDS);
5416		- }
5417		- }
5418		-}
5419	3057
5420	3058	public function mxchat_handle_post_delete($post_id) {
5421	3059	// Get post data before it's deleted
5422	3060	$post = get_post($post_id);
		@@ -5450,17 +3088,33 @@
5450	3088
5451	3089	// Get the URL before post is deleted
5452	3090	$source_url = get_permalink($post_id);
5453	3091	if (!$source_url) {
5454		- //error_log('MXChat: Failed to get permalink for post ' . $post_id);
	3092	+ error_log('MXChat: Failed to get permalink for post ' . $post_id);
5455	3093	return;
5456	3094	}
5457	3095
5458		- // Use chunk-aware deletion (handles both chunked and non-chunked content)
5459		- $delete_result = MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	3096	+ // Check if Pinecone is enabled
	3097	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	3098	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5460	3099
5461		- if (is_wp_error($delete_result)) {
5462		- //error_log('MXChat: Chunk-aware deletion failed for URL: ' . $source_url . ' - ' . $delete_result->get_error_message());
	3100	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3101	+ // Delete from Pinecone
	3102	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
	3103	+ } else {
	3104	+ // Delete from WordPress DB
	3105	+ global $wpdb;
	3106	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	3107	+
	3108	+ $result = $wpdb->delete(
	3109	+ $table_name,
	3110	+ array('source_url' => $source_url),
	3111	+ array('%s')
	3112	+ );
	3113	+
	3114	+ if ($result === false) {
	3115	+ error_log('MXChat: WordPress DB deletion failed for URL: ' . $source_url);
	3116	+ }
5463	3117	}
5464	3118	}
5465	3119
5466	3120
		@@ -5531,12 +3185,11 @@
5531	3185	if (!isset($this->options['enable_woocommerce_integration']) ||
5532	3186	!in_array($this->options['enable_woocommerce_integration'], ['1', 'on'])) {
5533	3187	return;
5534	3188	}
5535		-
	3189	+
5536	3190	$source_url = get_permalink($product->get_id());
5537		- $product_id = $product->get_id();
5538		-
	3191	+
5539	3192	// Build product content
5540	3193	$title = $product->get_name();
5541	3194	$description = $product->get_description();
5542	3195	$short_description = $product->get_short_description();
		@@ -5541,92 +3194,35 @@
5541	3194	$description = $product->get_description();
5542	3195	$short_description = $product->get_short_description();
5543	3196	$regular_price = $product->get_regular_price();
5544	3197	$sale_price = $product->get_sale_price();
5545		- $price = $product->get_price();
5546	3198	$sku = $product->get_sku();
5547		-
5548		- // Get currency symbol
5549		- $currency_symbol = get_woocommerce_currency_symbol();
5550		-
	3199	+
5551	3200	// Format content consistently
5552	3201	$content = $title . "\n\n";
5553		-
5554		- if (!empty($short_description)) {
5555		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
5556		- }
5557		-
	3202	+
5558	3203	if (!empty($description)) {
5559	3204	$content .= wp_strip_all_tags($description) . "\n\n";
5560	3205	}
5561		-
5562		- // Add pricing information
5563		- if (!empty($regular_price)) {
5564		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
5565		- } elseif (!empty($price)) {
5566		- $content .= "Price: " . $currency_symbol . $price . "\n";
	3206	+
	3207	+ if (!empty($short_description)) {
	3208	+ $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
5567	3209	}
5568		-
5569		- if (!empty($sale_price) && $sale_price !== $regular_price) {
5570		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
	3210	+
	3211	+ $content .= "Price: $" . $regular_price . "\n";
	3212	+
	3213	+ if (!empty($sale_price)) {
	3214	+ $content .= "Sale Price: $" . $sale_price . "\n";
5571	3215	}
5572		-
5573		- // Handle variable products - show price range
5574		- if ($product->is_type('variable')) {
5575		- $min_price = $product->get_variation_price('min');
5576		- $max_price = $product->get_variation_price('max');
5577		- if ($min_price !== $max_price) {
5578		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
5579		- }
5580		- }
5581		-
	3216	+
5582	3217	if (!empty($sku)) {
5583	3218	$content .= "SKU: " . $sku . "\n";
5584	3219	}
5585		-
5586		- // Get product categories
5587		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
5588		- if (!empty($categories) && !is_wp_error($categories)) {
5589		- $content .= "Categories: " . implode(', ', $categories) . "\n";
5590		- }
5591		-
5592		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
5593		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
5594		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
5595		- foreach ($custom_tabs as $tab) {
5596		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
5597		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
5598		-
5599		- if (!empty($tab_title) && !empty($tab_content)) {
5600		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
5601		- }
5602		- }
5603		- }
5604		-
5605		- // Also check for reusable/saved tabs applied to this product
5606		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
5607		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
5608		- // Get the saved tabs option
5609		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
5610		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
5611		- foreach ($applied_saved_tabs as $saved_tab_id) {
5612		- if (isset($saved_tabs[$saved_tab_id])) {
5613		- $tab = $saved_tabs[$saved_tab_id];
5614		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
5615		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
5616		-
5617		- if (!empty($tab_title) && !empty($tab_content)) {
5618		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
5619		- }
5620		- }
5621		- }
5622		- }
5623		- }
5624		-
	3220	+
5625	3221	// Get API key with proper model detection
5626	3222	$options = get_option('mxchat_options');
5627	3223	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
5628		-
	3224	+
5629	3225	if (strpos($selected_model, 'voyage') === 0) {
5630	3226	$api_key = $options['voyage_api_key'] ?? '';
5631	3227	} elseif (strpos($selected_model, 'gemini-embedding') === 0) {
5632	3228	$api_key = $options['gemini_api_key'] ?? '';
		@@ -5632,14 +3228,14 @@
5632	3228	$api_key = $options['gemini_api_key'] ?? '';
5633	3229	} else {
5634	3230	$api_key = $options['api_key'] ?? '';
5635	3231	}
5636		-
	3232	+
5637	3233	if (empty($api_key)) {
5638		- //error_log('MxChat Auto-sync: No API key configured for embedding model');
	3234	+ error_log('MxChat Auto-sync: No API key configured for embedding model');
5639	3235	return;
5640	3236	}
5641		-
	3237	+
5642	3238	// Use the centralized utility function for storage
5643	3239	$result = MxChat_Utils::submit_content_to_db(
5644	3240	$content,
5645	3241	$source_url,
		@@ -5645,16 +3241,11 @@
5645	3241	$source_url,
5646	3242	$api_key,
5647	3243	md5($source_url) // Vector ID for Pinecone
5648	3244	);
5649		-
5650		- // After successful storage, apply role restriction based on tags
5651		- if (!is_wp_error($result)) {
5652		- $this->apply_role_restriction_to_post($product_id, $source_url);
5653		- }
5654		-
	3245	+
5655	3246	if (is_wp_error($result)) {
5656		- //error_log('MxChat WooCommerce sync failed for product ' . $product_id . ': ' . $result->get_error_message());
	3247	+ error_log('MxChat WooCommerce sync failed for product ' . $product->get_id() . ': ' . $result->get_error_message());
5657	3248	}
5658	3249	}
5659	3250
5660	3251	public function mxchat_handle_product_delete($post_id) {
		@@ -5682,1101 +3273,9 @@
5682	3273	array('%s')
5683	3274	);
5684	3275	}
5685	3276	}
5686		-
5687		-/**
5688		- * Handle individual Pinecone content deletion
5689		- */
5690		-public function mxchat_handle_pinecone_prompt_delete() {
5691		- // Check permissions
5692		- if (!current_user_can('manage_options')) {
5693		- wp_die(esc_html__('You do not have sufficient permissions.', 'mxchat'));
5694		- }
5695	3277
5696		- // Verify nonce
5697		- if (!isset($_GET['_wpnonce']) || !wp_verify_nonce($_GET['_wpnonce'], 'mxchat_delete_pinecone_prompt_nonce')) {
5698		- wp_die(esc_html__('Security check failed.', 'mxchat'));
5699		- }
5700		-
5701		- $vector_id = isset($_GET['vector_id']) ? sanitize_text_field($_GET['vector_id']) : '';
5702		-
5703		- if (empty($vector_id)) {
5704		- set_transient('mxchat_admin_notice_error',
5705		- esc_html__('Invalid vector ID.', 'mxchat'),
5706		- 30
5707		- );
5708		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
5709		- exit;
5710		- }
5711		-
5712		- // Get Pinecone settings
5713		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
5714		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5715		-
5716		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
5717		- set_transient('mxchat_admin_notice_error',
5718		- esc_html__('Pinecone is not properly configured.', 'mxchat'),
5719		- 30
5720		- );
5721		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
5722		- exit;
5723		- }
5724		-
5725		- // Delete from Pinecone
5726		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
5727		- $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
5728		- $vector_id,
5729		- $pinecone_options['mxchat_pinecone_api_key'],
5730		- $pinecone_options['mxchat_pinecone_host']
5731		- );
5732		-
5733		- if ($result['success']) {
5734		- // No cache clearing needed since we removed caching
5735		- set_transient('mxchat_admin_notice_success',
5736		- esc_html__('Entry deleted successfully from Pinecone.', 'mxchat'),
5737		- 30
5738		- );
5739		- } else {
5740		- set_transient('mxchat_admin_notice_error',
5741		- esc_html__('Failed to delete entry: ', 'mxchat') . $result['message'],
5742		- 30
5743		- );
5744		- }
5745		-
5746		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
5747		- exit;
5748		-}
5749		-/**
5750		- * Handle individual Pinecone content deletion via AJAX
5751		- */
5752		-public function ajax_mxchat_delete_pinecone_prompt() {
5753		- // Verify nonce and permissions
5754		- if (!check_ajax_referer('mxchat_delete_pinecone_prompt_nonce', 'nonce', false)) {
5755		- wp_send_json_error('Invalid nonce');
5756		- exit;
5757		- }
5758		-
5759		- if (!current_user_can('manage_options')) {
5760		- wp_send_json_error('Unauthorized access');
5761		- exit;
5762		- }
5763		-
5764		- $vector_id = isset($_POST['vector_id']) ? sanitize_text_field($_POST['vector_id']) : '';
5765		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
5766		-
5767		- if (empty($vector_id)) {
5768		- wp_send_json_error('Missing vector ID');
5769		- exit;
5770		- }
5771		-
5772		- // Get bot-specific Pinecone settings
5773		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
5774		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
5775		-
5776		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5777		-
5778		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
5779		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
5780		- exit;
5781		- }
5782		-
5783		- // Delete from the correct Pinecone index
5784		- $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
5785		- $vector_id,
5786		- $pinecone_options['mxchat_pinecone_api_key'],
5787		- $pinecone_options['mxchat_pinecone_host']
5788		- );
5789		-
5790		- if ($result['success']) {
5791		- // No cache clearing needed since we removed caching
5792		- wp_send_json_success(array(
5793		- 'message' => 'Entry deleted successfully from Pinecone',
5794		- 'vector_id' => $vector_id,
5795		- 'bot_id' => $bot_id
5796		- ));
5797		- } else {
5798		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete from Pinecone: ' . $result['message'], array('vector_id' => $vector_id, 'bot_id' => $bot_id));
5799		- wp_send_json_error('Failed to delete from Pinecone: ' . $result['message']);
5800		- }
5801		-
5802		- exit;
5803		-}
5804		-
5805		-/**
5806		- * Handle deletion of all chunks for a given source URL via AJAX
5807		- * Follows the same pattern as ajax_mxchat_delete_pinecone_prompt
5808		- */
5809		-public function ajax_mxchat_delete_chunks_by_url() {
5810		- // Verify nonce and permissions
5811		- if (!check_ajax_referer('mxchat_delete_chunks_nonce', 'nonce', false)) {
5812		- wp_send_json_error('Invalid nonce');
5813		- exit;
5814		- }
5815		-
5816		- if (!current_user_can('manage_options')) {
5817		- wp_send_json_error('Unauthorized access');
5818		- exit;
5819		- }
5820		-
5821		- $source_url = isset($_POST['source_url']) ? esc_url_raw($_POST['source_url']) : '';
5822		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
5823		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
5824		-
5825		- if (empty($source_url)) {
5826		- wp_send_json_error('Missing source URL');
5827		- exit;
5828		- }
5829		-
5830		- // Generate the base vector ID from the source URL (same as how chunks are created)
5831		- $base_vector_id = md5($source_url);
5832		-
5833		- if ($data_source === 'pinecone') {
5834		- // Get bot-specific Pinecone settings (same as working delete function)
5835		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
5836		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
5837		-
5838		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5839		-
5840		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
5841		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
5842		- exit;
5843		- }
5844		-
5845		- $api_key = $pinecone_options['mxchat_pinecone_api_key'];
5846		- $host = $pinecone_options['mxchat_pinecone_host'];
5847		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
5848		-
5849		- // Collect all vector IDs to delete
5850		- $vectors_to_delete = array();
5851		-
5852		- // Add the original single-vector ID (for non-chunked content)
5853		- $vectors_to_delete[] = $base_vector_id;
5854		-
5855		- // Use Pinecone list API to find all chunk vectors with this prefix
5856		- // NOTE: Pinecone List API is a GET request with query parameters, not POST
5857		- $prefix = $base_vector_id . '_chunk_';
5858		-
5859		- $query_params = array(
5860		- 'prefix' => $prefix,
5861		- 'limit' => 100
5862		- );
5863		-
5864		- if (!empty($namespace)) {
5865		- $query_params['namespace'] = $namespace;
5866		- }
5867		-
5868		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
5869		-
5870		- $list_response = wp_remote_get($list_url, array(
5871		- 'headers' => array(
5872		- 'Api-Key' => $api_key,
5873		- 'accept' => 'application/json'
5874		- ),
5875		- 'timeout' => 30
5876		- ));
5877		-
5878		- if (!is_wp_error($list_response)) {
5879		- $list_body_response = wp_remote_retrieve_body($list_response);
5880		- $list_data = json_decode($list_body_response, true);
5881		- if (!empty($list_data['vectors'])) {
5882		- foreach ($list_data['vectors'] as $vector) {
5883		- if (isset($vector['id'])) {
5884		- $vectors_to_delete[] = $vector['id'];
5885		- }
5886		- }
5887		- }
5888		- }
5889		-
5890		- if (empty($vectors_to_delete)) {
5891		- wp_send_json_success(array(
5892		- 'message' => 'No vectors found to delete',
5893		- 'source_url' => $source_url
5894		- ));
5895		- exit;
5896		- }
5897		-
5898		- // Delete all vectors using the same endpoint as the working function
5899		- $delete_url = "https://{$host}/vectors/delete";
5900		-
5901		- $delete_body = array(
5902		- 'ids' => $vectors_to_delete
5903		- );
5904		-
5905		- if (!empty($namespace)) {
5906		- $delete_body['namespace'] = $namespace;
5907		- }
5908		-
5909		- $delete_response = wp_remote_post($delete_url, array(
5910		- 'headers' => array(
5911		- 'Api-Key' => $api_key,
5912		- 'accept' => 'application/json',
5913		- 'content-type' => 'application/json'
5914		- ),
5915		- 'body' => wp_json_encode($delete_body),
5916		- 'timeout' => 30
5917		- ));
5918		-
5919		- if (is_wp_error($delete_response)) {
5920		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete chunks from Pinecone: ' . $delete_response->get_error_message(), array('source_url' => $source_url));
5921		- wp_send_json_error('Failed to delete from Pinecone: ' . $delete_response->get_error_message());
5922		- exit;
5923		- }
5924		-
5925		- $response_code = wp_remote_retrieve_response_code($delete_response);
5926		-
5927		- if ($response_code !== 200) {
5928		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Pinecone API error (HTTP ' . $response_code . ')', array('source_url' => $source_url));
5929		- wp_send_json_error('Pinecone API error (HTTP ' . $response_code . ')');
5930		- exit;
5931		- }
5932		-
5933		- wp_send_json_success(array(
5934		- 'message' => 'All chunks deleted successfully from Pinecone',
5935		- 'source_url' => $source_url,
5936		- 'deleted_count' => count($vectors_to_delete)
5937		- ));
5938		-
5939		- } else {
5940		- // WordPress database deletion
5941		- global $wpdb;
5942		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
5943		-
5944		- $result = $wpdb->delete(
5945		- $table_name,
5946		- array('source_url' => $source_url),
5947		- array('%s')
5948		- );
5949		-
5950		- if ($result === false) {
5951		- MxChat_Admin::mxchat_log_debug('database_error', 'Failed to delete from database: ' . $wpdb->last_error, array('source_url' => $source_url));
5952		- wp_send_json_error('Failed to delete from database: ' . $wpdb->last_error);
5953		- exit;
5954		- }
5955		-
5956		- wp_send_json_success(array(
5957		- 'message' => 'All chunks deleted successfully from database',
5958		- 'source_url' => $source_url,
5959		- 'deleted_count' => $result
5960		- ));
5961		- }
5962		-
5963		- exit;
5964		-}
5965		-
5966		-/**
5967		- * Handle individual WordPress database content deletion via AJAX
5968		- * Mirrors the Pinecone delete handler but for WordPress database entries
5969		- */
5970		-public function ajax_mxchat_delete_wordpress_prompt() {
5971		- // Verify nonce and permissions
5972		- if (!check_ajax_referer('mxchat_delete_wordpress_prompt_nonce', 'nonce', false)) {
5973		- wp_send_json_error('Invalid nonce');
5974		- exit;
5975		- }
5976		-
5977		- if (!current_user_can('manage_options')) {
5978		- wp_send_json_error('Unauthorized access');
5979		- exit;
5980		- }
5981		-
5982		- $entry_id = isset($_POST['entry_id']) ? intval($_POST['entry_id']) : 0;
5983		-
5984		- if (empty($entry_id)) {
5985		- wp_send_json_error('Missing entry ID');
5986		- exit;
5987		- }
5988		-
5989		- global $wpdb;
5990		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
5991		-
5992		- // Clear cache for this entry
5993		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
5994		-
5995		- // Delete from database
5996		- $result = $wpdb->delete(
5997		- $table_name,
5998		- array('id' => $entry_id),
5999		- array('%d')
6000		- );
6001		-
6002		- if ($result !== false) {
6003		- wp_send_json_success(array(
6004		- 'message' => 'Entry deleted successfully',
6005		- 'entry_id' => $entry_id
6006		- ));
6007		- } else {
6008		- wp_send_json_error('Failed to delete entry from database');
6009		- }
6010		-
6011		- exit;
6012		-}
6013		-
6014		-/**
6015		- * Handle bulk deletion of knowledge entries via AJAX
6016		- * Supports both Pinecone and WordPress database entries
6017		- */
6018		-public function ajax_mxchat_bulk_delete_knowledge() {
6019		- // Verify nonce and permissions
6020		- if (!check_ajax_referer('mxchat_bulk_delete_knowledge_nonce', 'nonce', false)) {
6021		- wp_send_json_error('Invalid nonce');
6022		- exit;
6023		- }
6024		-
6025		- if (!current_user_can('manage_options')) {
6026		- wp_send_json_error('Unauthorized access');
6027		- exit;
6028		- }
6029		-
6030		- $entries = isset($_POST['entries']) ? $_POST['entries'] : array();
6031		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6032		-
6033		- if (empty($entries) || !is_array($entries)) {
6034		- wp_send_json_error('No entries provided');
6035		- exit;
6036		- }
6037		-
6038		- // Extend execution time — bulk Pinecone operations can take a while
6039		- if (function_exists('set_time_limit')) {
6040		- set_time_limit(120);
6041		- }
6042		-
6043		- $success_ids = array();
6044		- $failed_ids = array();
6045		- $errors = array();
6046		-
6047		- global $wpdb;
6048		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6049		-
6050		- // Get Pinecone manager for Pinecone deletions
6051		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6052		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6053		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6054		-
6055		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
6056		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
6057		-
6058		- // =============================================
6059		- // PHASE 1: Collect all Pinecone vector IDs
6060		- // and separate WordPress entries
6061		- // =============================================
6062		- $pinecone_entry_ids = array(); // entry IDs that are pinecone-sourced
6063		- $wordpress_entries = array(); // entries for WordPress DB deletion
6064		- $all_vector_ids = array(); // all pinecone vector IDs to delete in one batch
6065		-
6066		- foreach ($entries as $entry) {
6067		- $entry_id = sanitize_text_field($entry['id'] ?? '');
6068		- $source = sanitize_text_field($entry['source'] ?? 'wordpress');
6069		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
6070		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
6071		-
6072		- if (empty($entry_id)) {
6073		- continue;
6074		- }
6075		-
6076		- if ($source === 'pinecone') {
6077		- if (!$use_pinecone || empty($api_key)) {
6078		- $failed_ids[] = $entry_id;
6079		- $errors[] = "Pinecone not configured for entry: $entry_id";
6080		- continue;
6081		- }
6082		-
6083		- $pinecone_entry_ids[] = $entry_id;
6084		-
6085		- if ($is_group && !empty($source_url)) {
6086		- // Grouped/chunked entry: collect base ID + chunk IDs via List API
6087		- $base_vector_id = md5($source_url);
6088		- $all_vector_ids[] = $base_vector_id;
6089		-
6090		- $list_url = 'https://' . $host . '/vectors/list?prefix=' . urlencode($base_vector_id . '_chunk_') . '&limit=100';
6091		- $list_response = wp_remote_get($list_url, array(
6092		- 'headers' => array(
6093		- 'Api-Key' => $api_key,
6094		- 'accept' => 'application/json'
6095		- ),
6096		- 'timeout' => 30
6097		- ));
6098		-
6099		- if (!is_wp_error($list_response)) {
6100		- $list_body = json_decode(wp_remote_retrieve_body($list_response), true);
6101		- if (!empty($list_body['vectors']) && is_array($list_body['vectors'])) {
6102		- foreach ($list_body['vectors'] as $vector) {
6103		- if (isset($vector['id'])) {
6104		- $all_vector_ids[] = $vector['id'];
6105		- }
6106		- }
6107		- }
6108		- }
6109		- } else {
6110		- // Single entry: the entry_id IS the vector ID
6111		- $all_vector_ids[] = $entry_id;
6112		- }
6113		- } else {
6114		- $wordpress_entries[] = $entry;
6115		- }
6116		- }
6117		-
6118		- // =============================================
6119		- // PHASE 2: Single batch delete to Pinecone
6120		- // =============================================
6121		- if (!empty($all_vector_ids)) {
6122		- $all_vector_ids = array_values(array_unique($all_vector_ids));
6123		- $pinecone_success = true;
6124		- $batches = array_chunk($all_vector_ids, 100);
6125		-
6126		- foreach ($batches as $batch) {
6127		- $delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
6128		- 'headers' => array(
6129		- 'Api-Key' => $api_key,
6130		- 'accept' => 'application/json',
6131		- 'content-type' => 'application/json'
6132		- ),
6133		- 'body' => wp_json_encode(array('ids' => $batch)),
6134		- 'timeout' => 60
6135		- ));
6136		-
6137		- if (is_wp_error($delete_response)) {
6138		- $pinecone_success = false;
6139		- $errors[] = 'Pinecone batch deletion failed: ' . $delete_response->get_error_message();
6140		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed: ' . $delete_response->get_error_message());
6141		- } else {
6142		- $response_code = wp_remote_retrieve_response_code($delete_response);
6143		- if ($response_code !== 200) {
6144		- $pinecone_success = false;
6145		- $response_body = wp_remote_retrieve_body($delete_response);
6146		- $errors[] = "Pinecone API error (HTTP $response_code)";
6147		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed (HTTP ' . $response_code . ')', array('response' => substr($response_body, 0, 200)));
6148		- }
6149		- }
6150		- }
6151		-
6152		- // Mark all pinecone entries based on batch result
6153		- foreach ($pinecone_entry_ids as $eid) {
6154		- if ($pinecone_success) {
6155		- $success_ids[] = $eid;
6156		- } else {
6157		- $failed_ids[] = $eid;
6158		- }
6159		- }
6160		- }
6161		-
6162		- // =============================================
6163		- // PHASE 3: WordPress database deletions
6164		- // =============================================
6165		- foreach ($wordpress_entries as $entry) {
6166		- $entry_id = sanitize_text_field($entry['id'] ?? '');
6167		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
6168		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
6169		-
6170		- if (empty($entry_id)) {
6171		- continue;
6172		- }
6173		-
6174		- try {
6175		- if ($is_group && !empty($source_url)) {
6176		- $result = $wpdb->delete(
6177		- $table_name,
6178		- array('source_url' => $source_url),
6179		- array('%s')
6180		- );
6181		- } else {
6182		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
6183		- $result = $wpdb->delete(
6184		- $table_name,
6185		- array('id' => intval($entry_id)),
6186		- array('%d')
6187		- );
6188		- }
6189		-
6190		- if ($result !== false) {
6191		- $success_ids[] = $entry_id;
6192		- } else {
6193		- $failed_ids[] = $entry_id;
6194		- $errors[] = "Database error for entry: $entry_id";
6195		- }
6196		- } catch (Exception $e) {
6197		- $failed_ids[] = $entry_id;
6198		- $errors[] = $e->getMessage();
6199		- }
6200		- }
6201		-
6202		- wp_send_json_success(array(
6203		- 'success_ids' => $success_ids,
6204		- 'failed_ids' => $failed_ids,
6205		- 'errors' => $errors,
6206		- 'total_processed' => count($success_ids) + count($failed_ids)
6207		- ));
6208		-
6209		- exit;
6210		-}
6211		-
6212		-/**
6213		- * Get hierarchical roles for dropdown
6214		- */
6215		-public function mxchat_get_role_options() {
6216		- return array(
6217		- 'public' => __('Public (Everyone)', 'mxchat'),
6218		- 'logged_in' => __('Logged In Users', 'mxchat'),
6219		- 'subscriber' => __('Subscribers & Above', 'mxchat'),
6220		- 'contributor' => __('Contributors & Above', 'mxchat'),
6221		- 'author' => __('Authors & Above', 'mxchat'),
6222		- 'editor' => __('Editors & Above', 'mxchat'),
6223		- 'administrator' => __('Administrators Only', 'mxchat')
6224		- );
6225		-}
6226		-
6227		-/**
6228		- * Check if user has access to content based on role restriction
6229		- */
6230		-public function mxchat_user_has_content_access($role_restriction) {
6231		- // Public content is always accessible
6232		- if ($role_restriction === 'public' || empty($role_restriction)) {
6233		- return true;
6234		- }
6235		-
6236		- // Check if user is logged in for logged_in restriction
6237		- if ($role_restriction === 'logged_in') {
6238		- return is_user_logged_in();
6239		- }
6240		-
6241		- // If not logged in, no access to role-restricted content
6242		- if (!is_user_logged_in()) {
6243		- return false;
6244		- }
6245		-
6246		- $user = wp_get_current_user();
6247		- $user_roles = $user->roles;
6248		-
6249		- if (empty($user_roles)) {
6250		- return false;
6251		- }
6252		-
6253		- // Define role hierarchy (higher number = higher access)
6254		- $hierarchy = array(
6255		- 'subscriber' => 1,
6256		- 'contributor' => 2,
6257		- 'author' => 3,
6258		- 'editor' => 4,
6259		- 'administrator' => 5
6260		- );
6261		-
6262		- // Get required level
6263		- $required_level = isset($hierarchy[$role_restriction]) ? $hierarchy[$role_restriction] : 0;
6264		-
6265		- // Check if user has required level or higher
6266		- foreach ($user_roles as $user_role) {
6267		- $user_level = isset($hierarchy[$user_role]) ? $hierarchy[$user_role] : 0;
6268		- if ($user_level >= $required_level) {
6269		- return true;
6270		- }
6271		- }
6272		-
6273		- return false;
6274		-}
6275		-
6276		-/**
6277		- * Handle role restriction updates via AJAX
6278		- * Removed cache clearing call since we removed caching
6279		- */
6280		-public function ajax_mxchat_update_role_restriction() {
6281		- // Verify nonce and permissions
6282		- if (!check_ajax_referer('mxchat_update_role_nonce', 'nonce', false)) {
6283		- wp_send_json_error('Invalid nonce');
6284		- exit;
6285		- }
6286		-
6287		- if (!current_user_can('manage_options')) {
6288		- wp_send_json_error('Unauthorized access');
6289		- exit;
6290		- }
6291		-
6292		- $entry_id = isset($_POST['entry_id']) ? sanitize_text_field($_POST['entry_id']) : '';
6293		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
6294		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
6295		-
6296		- if (empty($entry_id)) {
6297		- wp_send_json_error('Invalid entry ID');
6298		- exit;
6299		- }
6300		-
6301		- // Get knowledge manager instance to validate role restriction
6302		- $knowledge_manager = MxChat_Knowledge_Manager::get_instance();
6303		- $valid_roles = array_keys($knowledge_manager->mxchat_get_role_options());
6304		- if (!in_array($role_restriction, $valid_roles)) {
6305		- wp_send_json_error('Invalid role restriction');
6306		- exit;
6307		- }
6308		-
6309		- global $wpdb;
6310		-
6311		- if ($data_source === 'pinecone') {
6312		- // Handle Pinecone role restriction (stored separately in WordPress table)
6313		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6314		-
6315		- // Use REPLACE to insert or update the role restriction
6316		- $result = $wpdb->replace(
6317		- $roles_table,
6318		- array(
6319		- 'vector_id' => $entry_id,
6320		- 'role_restriction' => $role_restriction,
6321		- 'updated_at' => current_time('mysql')
6322		- ),
6323		- array('%s', '%s', '%s')
6324		- );
6325		-
6326		- // No cache clearing needed since we removed caching
6327		-
6328		- } else {
6329		- // Handle WordPress database role restriction (existing functionality)
6330		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6331		-
6332		- $result = $wpdb->update(
6333		- $table_name,
6334		- array('role_restriction' => $role_restriction),
6335		- array('id' => absint($entry_id)),
6336		- array('%s'),
6337		- array('%d')
6338		- );
6339		- }
6340		-
6341		- if ($result === false) {
6342		- wp_send_json_error('Database update failed: ' . $wpdb->last_error);
6343		- exit;
6344		- }
6345		-
6346		- wp_send_json_success(array(
6347		- 'message' => 'Role restriction updated successfully',
6348		- 'role_restriction' => $role_restriction,
6349		- 'data_source' => $data_source,
6350		- 'entry_id' => $entry_id
6351		- ));
6352		- exit;
6353		-}
6354		-
6355		-// ========================================
6356		-// ROLE-BASED CONTENT RESTRICTIONS - NEW FUNCTIONS
6357		-// Add these to your MxChat_Knowledge_Manager class
6358		-// ========================================
6359		-
6360		-/**
6361		- * Initialize role-based content hooks
6362		- * Add this call to your __construct() or mxchat_init_hooks() method
6363		- */
6364		-private function mxchat_init_role_hooks() {
6365		- // AJAX handlers for tag-role mappings
6366		- add_action('wp_ajax_mxchat_add_tag_role_mapping', array($this, 'ajax_add_tag_role_mapping'));
6367		- add_action('wp_ajax_mxchat_delete_tag_role_mapping', array($this, 'ajax_delete_tag_role_mapping'));
6368		- add_action('wp_ajax_mxchat_get_tag_role_mappings', array($this, 'ajax_get_tag_role_mappings'));
6369		- add_action('wp_ajax_mxchat_bulk_update_tag_roles', array($this, 'ajax_bulk_update_tag_roles'));
6370		-
6371		- // Hook to automatically update role restrictions when tags are added/removed
6372		- add_action('set_object_terms', array($this, 'handle_tag_change'), 10, 6);
6373		-
6374		- // Hook to apply role restrictions on auto-sync
6375		- add_action('mxchat_content_stored', array($this, 'apply_role_restriction_after_storage'), 10, 2);
6376		-}
6377		-
6378		-/**
6379		- * Add tag-role mapping via AJAX
6380		- */
6381		-public function ajax_add_tag_role_mapping() {
6382		- // Verify nonce and permissions
6383		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6384		-
6385		- if (!current_user_can('manage_options')) {
6386		- wp_send_json_error('Unauthorized access');
6387		- exit;
6388		- }
6389		-
6390		- $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
6391		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
6392		-
6393		- if (empty($tag_slug)) {
6394		- wp_send_json_error('Tag slug is required');
6395		- exit;
6396		- }
6397		-
6398		- // Validate role restriction
6399		- $valid_roles = array_keys($this->mxchat_get_role_options());
6400		- if (!in_array($role_restriction, $valid_roles)) {
6401		- wp_send_json_error('Invalid role restriction');
6402		- exit;
6403		- }
6404		-
6405		- // Check if tag exists in WordPress
6406		- $term = get_term_by('slug', $tag_slug, 'post_tag');
6407		- if (!$term) {
6408		- wp_send_json_error('Tag does not exist in WordPress');
6409		- exit;
6410		- }
6411		-
6412		- // Get existing mappings
6413		- $mappings = get_option('mxchat_tag_role_mappings', array());
6414		-
6415		- // Check if mapping already exists
6416		- if (isset($mappings[$tag_slug])) {
6417		- wp_send_json_error('Mapping for this tag already exists');
6418		- exit;
6419		- }
6420		-
6421		- // Add new mapping
6422		- $mappings[$tag_slug] = $role_restriction;
6423		- update_option('mxchat_tag_role_mappings', $mappings);
6424		-
6425		- wp_send_json_success(array(
6426		- 'message' => 'Tag-role mapping added successfully',
6427		- 'tag_slug' => $tag_slug,
6428		- 'role_restriction' => $role_restriction
6429		- ));
6430		- exit;
6431		-}
6432		-
6433		-/**
6434		- * Delete tag-role mapping via AJAX
6435		- */
6436		-public function ajax_delete_tag_role_mapping() {
6437		- // Verify nonce and permissions
6438		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6439		-
6440		- if (!current_user_can('manage_options')) {
6441		- wp_send_json_error('Unauthorized access');
6442		- exit;
6443		- }
6444		-
6445		- $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
6446		-
6447		- if (empty($tag_slug)) {
6448		- wp_send_json_error('Tag slug is required');
6449		- exit;
6450		- }
6451		-
6452		- // Get existing mappings
6453		- $mappings = get_option('mxchat_tag_role_mappings', array());
6454		-
6455		- // Check if mapping exists
6456		- if (!isset($mappings[$tag_slug])) {
6457		- wp_send_json_error('Mapping does not exist');
6458		- exit;
6459		- }
6460		-
6461		- // Remove mapping
6462		- unset($mappings[$tag_slug]);
6463		- update_option('mxchat_tag_role_mappings', $mappings);
6464		-
6465		- wp_send_json_success(array(
6466		- 'message' => 'Tag-role mapping deleted successfully',
6467		- 'tag_slug' => $tag_slug
6468		- ));
6469		- exit;
6470		-}
6471		-
6472		-/**
6473		- * Get all tag-role mappings via AJAX
6474		- */
6475		-public function ajax_get_tag_role_mappings() {
6476		- // Verify nonce and permissions
6477		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6478		-
6479		- if (!current_user_can('manage_options')) {
6480		- wp_send_json_error('Unauthorized access');
6481		- exit;
6482		- }
6483		-
6484		- // Get mappings
6485		- $mappings = get_option('mxchat_tag_role_mappings', array());
6486		- $role_options = $this->mxchat_get_role_options();
6487		-
6488		- $formatted_mappings = array();
6489		-
6490		- foreach ($mappings as $tag_slug => $role_restriction) {
6491		- // Get tag object
6492		- $term = get_term_by('slug', $tag_slug, 'post_tag');
6493		-
6494		- // Count posts with this tag
6495		- $post_count = 0;
6496		- if ($term) {
6497		- $post_count = $term->count;
6498		- }
6499		-
6500		- $formatted_mappings[] = array(
6501		- 'tag_slug' => $tag_slug,
6502		- 'role_restriction' => $role_restriction,
6503		- 'role_label' => $role_options[$role_restriction] ?? $role_restriction,
6504		- 'post_count' => $post_count
6505		- );
6506		- }
6507		-
6508		- wp_send_json_success(array(
6509		- 'mappings' => $formatted_mappings
6510		- ));
6511		- exit;
6512		-}
6513		-
6514		-/**
6515		- * Bulk update role restrictions for all existing content with mapped tags
6516		- */
6517		-public function ajax_bulk_update_tag_roles() {
6518		- // Verify nonce and permissions
6519		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6520		-
6521		- if (!current_user_can('manage_options')) {
6522		- wp_send_json_error('Unauthorized access');
6523		- exit;
6524		- }
6525		-
6526		- // Get mappings
6527		- $mappings = get_option('mxchat_tag_role_mappings', array());
6528		-
6529		- if (empty($mappings)) {
6530		- wp_send_json_error('No tag-role mappings found');
6531		- exit;
6532		- }
6533		-
6534		- global $wpdb;
6535		-
6536		- // Check if using Pinecone
6537		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6538		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6539		-
6540		- $updated_count = 0;
6541		- $details = array();
6542		-
6543		- foreach ($mappings as $tag_slug => $role_restriction) {
6544		- // Get all posts with this tag
6545		- $posts = get_posts(array(
6546		- 'tag' => $tag_slug,
6547		- 'post_type' => 'any',
6548		- 'posts_per_page' => -1,
6549		- 'fields' => 'ids',
6550		- 'post_status' => 'publish'
6551		- ));
6552		-
6553		- if (empty($posts)) {
6554		- continue;
6555		- }
6556		-
6557		- $tag_updated = 0;
6558		-
6559		- foreach ($posts as $post_id) {
6560		- $source_url = get_permalink($post_id);
6561		- if (!$source_url) {
6562		- continue;
6563		- }
6564		-
6565		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
6566		- // Update Pinecone role restriction
6567		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6568		- $vector_id = md5($source_url);
6569		-
6570		- $result = $wpdb->replace(
6571		- $roles_table,
6572		- array(
6573		- 'vector_id' => $vector_id,
6574		- 'role_restriction' => $role_restriction,
6575		- 'updated_at' => current_time('mysql')
6576		- ),
6577		- array('%s', '%s', '%s')
6578		- );
6579		- } else {
6580		- // Update WordPress DB
6581		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6582		-
6583		- $result = $wpdb->update(
6584		- $table_name,
6585		- array('role_restriction' => $role_restriction),
6586		- array('source_url' => $source_url),
6587		- array('%s'),
6588		- array('%s')
6589		- );
6590		- }
6591		-
6592		- if ($result !== false) {
6593		- $tag_updated++;
6594		- $updated_count++;
6595		- }
6596		- }
6597		-
6598		- if ($tag_updated > 0) {
6599		- $details[] = sprintf(
6600		- 'Tag "%s" (%s): %d posts updated',
6601		- $tag_slug,
6602		- $role_restriction,
6603		- $tag_updated
6604		- );
6605		- }
6606		- }
6607		-
6608		- wp_send_json_success(array(
6609		- 'message' => 'Bulk update completed',
6610		- 'updated_count' => $updated_count,
6611		- 'tags_processed' => count($mappings),
6612		- 'details' => $details
6613		- ));
6614		- exit;
6615		-}
6616		-
6617		-/**
6618		- * Handle tag changes on posts (when tags are added or removed)
6619		- */
6620		-public function handle_tag_change($object_id, $terms, $tt_ids, $taxonomy, $append, $old_tt_ids) {
6621		- // Only process post tags
6622		- if ($taxonomy !== 'post_tag') {
6623		- return;
6624		- }
6625		-
6626		- // Get tag-role mappings
6627		- $mappings = get_option('mxchat_tag_role_mappings', array());
6628		-
6629		- if (empty($mappings)) {
6630		- return;
6631		- }
6632		-
6633		- // Get the post's URL
6634		- $source_url = get_permalink($object_id);
6635		- if (!$source_url) {
6636		- return;
6637		- }
6638		-
6639		- // Determine the highest role restriction based on tags
6640		- $highest_role = 'public';
6641		- $role_hierarchy = array(
6642		- 'public' => 0,
6643		- 'logged_in' => 1,
6644		- 'subscriber' => 2,
6645		- 'contributor' => 3,
6646		- 'author' => 4,
6647		- 'editor' => 5,
6648		- 'administrator' => 6
6649		- );
6650		-
6651		- // Get all current tags for the post
6652		- $current_tags = wp_get_post_tags($object_id, array('fields' => 'slugs'));
6653		-
6654		- // Find the highest role restriction among the tags
6655		- foreach ($current_tags as $tag_slug) {
6656		- if (isset($mappings[$tag_slug])) {
6657		- $role = $mappings[$tag_slug];
6658		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
6659		- $highest_role = $role;
6660		- }
6661		- }
6662		- }
6663		-
6664		- // Update the role restriction in the database
6665		- global $wpdb;
6666		-
6667		- // Check if using Pinecone
6668		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6669		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6670		-
6671		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
6672		- // Update Pinecone role restriction
6673		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6674		- $vector_id = md5($source_url);
6675		-
6676		- $wpdb->replace(
6677		- $roles_table,
6678		- array(
6679		- 'vector_id' => $vector_id,
6680		- 'role_restriction' => $highest_role,
6681		- 'updated_at' => current_time('mysql')
6682		- ),
6683		- array('%s', '%s', '%s')
6684		- );
6685		- } else {
6686		- // Update WordPress DB
6687		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6688		-
6689		- $wpdb->update(
6690		- $table_name,
6691		- array('role_restriction' => $highest_role),
6692		- array('source_url' => $source_url),
6693		- array('%s'),
6694		- array('%s')
6695		- );
6696		- }
6697		-}
6698		-
6699		-/**
6700		- * Apply role restriction after content is stored (for auto-sync)
6701		- */
6702		-public function apply_role_restriction_after_storage($post_id, $source_url) {
6703		- // Get tag-role mappings
6704		- $mappings = get_option('mxchat_tag_role_mappings', array());
6705		-
6706		- if (empty($mappings)) {
6707		- return;
6708		- }
6709		-
6710		- // Get all tags for the post
6711		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
6712		-
6713		- if (empty($post_tags)) {
6714		- return;
6715		- }
6716		-
6717		- // Determine the highest role restriction based on tags
6718		- $highest_role = 'public';
6719		- $role_hierarchy = array(
6720		- 'public' => 0,
6721		- 'logged_in' => 1,
6722		- 'subscriber' => 2,
6723		- 'contributor' => 3,
6724		- 'author' => 4,
6725		- 'editor' => 5,
6726		- 'administrator' => 6
6727		- );
6728		-
6729		- foreach ($post_tags as $tag_slug) {
6730		- if (isset($mappings[$tag_slug])) {
6731		- $role = $mappings[$tag_slug];
6732		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
6733		- $highest_role = $role;
6734		- }
6735		- }
6736		- }
6737		-
6738		- // If no restricted tags found, return (leave as public)
6739		- if ($highest_role === 'public') {
6740		- return;
6741		- }
6742		-
6743		- // Update the role restriction
6744		- global $wpdb;
6745		-
6746		- // Check if using Pinecone
6747		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6748		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6749		-
6750		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
6751		- // Update Pinecone role restriction
6752		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6753		- $vector_id = md5($source_url);
6754		-
6755		- $wpdb->replace(
6756		- $roles_table,
6757		- array(
6758		- 'vector_id' => $vector_id,
6759		- 'role_restriction' => $highest_role,
6760		- 'updated_at' => current_time('mysql')
6761		- ),
6762		- array('%s', '%s', '%s')
6763		- );
6764		- } else {
6765		- // Update WordPress DB
6766		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6767		-
6768		- $wpdb->update(
6769		- $table_name,
6770		- array('role_restriction' => $highest_role),
6771		- array('source_url' => $source_url),
6772		- array('%s'),
6773		- array('%s')
6774		- );
6775		- }
6776		-}
6777		-
6778		-
6779	3278	// ========================================
6780	3279	// HELPER METHODS
6781	3280	// ========================================
6782	3281
		@@ -6843,1074 +3342,8 @@
6843	3342	*/
6844	3343	private function mxchat_get_pinecone_manager() {
6845	3344	return MxChat_Pinecone_Manager::get_instance();
6846	3345	}
6847		-
6848		-
6849		- // ========================================
6850		-// DATABASE QUEUE TABLE MANAGEMENT
6851		-// ========================================
6852		-
6853		-/**
6854		- * Create queue table on plugin activation
6855		- * Call this from your plugin activation hook
6856		- */
6857		-public function mxchat_create_queue_table() {
6858		- global $wpdb;
6859		-
6860		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
6861		- $charset_collate = $wpdb->get_charset_collate();
6862		-
6863		- $sql = "CREATE TABLE IF NOT EXISTS $table_name (
6864		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
6865		- queue_id varchar(64) NOT NULL,
6866		- item_type varchar(20) NOT NULL,
6867		- item_data longtext NOT NULL,
6868		- status varchar(20) NOT NULL DEFAULT 'pending',
6869		- bot_id varchar(50) NOT NULL DEFAULT 'default',
6870		- priority int(11) NOT NULL DEFAULT 0,
6871		- attempts int(11) NOT NULL DEFAULT 0,
6872		- max_attempts int(11) NOT NULL DEFAULT 3,
6873		- error_message text DEFAULT NULL,
6874		- created_at datetime NOT NULL,
6875		- started_at datetime DEFAULT NULL,
6876		- completed_at datetime DEFAULT NULL,
6877		- PRIMARY KEY (id),
6878		- KEY queue_id (queue_id),
6879		- KEY status (status),
6880		- KEY item_type (item_type),
6881		- KEY priority (priority)
6882		- ) $charset_collate;";
6883		-
6884		- require_once(ABSPATH . 'wp-admin/includes/upgrade.php');
6885		- dbDelta($sql);
6886		-
6887		- // Also create a meta table for queue metadata
6888		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
6889		-
6890		- $meta_sql = "CREATE TABLE IF NOT EXISTS $meta_table (
6891		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
6892		- queue_id varchar(64) NOT NULL,
6893		- meta_key varchar(255) NOT NULL,
6894		- meta_value longtext,
6895		- PRIMARY KEY (id),
6896		- KEY queue_id (queue_id),
6897		- KEY meta_key (meta_key)
6898		- ) $charset_collate;";
6899		-
6900		- dbDelta($meta_sql);
6901		-}
6902		-
6903		-/**
6904		- * Add items to the processing queue
6905		- *
6906		- * @param string $queue_id Unique identifier for this queue batch
6907		- * @param string $item_type Type of item (url, pdf_page)
6908		- * @param array $items Array of items to queue
6909		- * @param string $bot_id Bot ID for processing
6910		- * @return int Number of items queued
6911		- */
6912		-private function mxchat_add_to_queue($queue_id, $item_type, $items, $bot_id = 'default') {
6913		- global $wpdb;
6914		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
6915		-
6916		- $queued_count = 0;
6917		- $priority = 0;
6918		-
6919		- foreach ($items as $item) {
6920		- $result = $wpdb->insert(
6921		- $table_name,
6922		- array(
6923		- 'queue_id' => $queue_id,
6924		- 'item_type' => $item_type,
6925		- 'item_data' => wp_json_encode($item),
6926		- 'status' => 'pending',
6927		- 'bot_id' => $bot_id,
6928		- 'priority' => $priority,
6929		- 'attempts' => 0,
6930		- 'max_attempts' => 3,
6931		- 'created_at' => current_time('mysql')
6932		- ),
6933		- array('%s', '%s', '%s', '%s', '%s', '%d', '%d', '%d', '%s')
6934		- );
6935		-
6936		- if ($result) {
6937		- $queued_count++;
6938		- }
6939		-
6940		- $priority++; // Process in order
6941		- }
6942		-
6943		- return $queued_count;
6944		-}
6945		-
6946		-/**
6947		- * Store queue metadata (total counts, source URL, etc.)
6948		- */
6949		-private function mxchat_set_queue_meta($queue_id, $meta_key, $meta_value) {
6950		- global $wpdb;
6951		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
6952		-
6953		- // Check if meta exists
6954		- $existing = $wpdb->get_var($wpdb->prepare(
6955		- "SELECT id FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
6956		- $queue_id,
6957		- $meta_key
6958		- ));
6959		-
6960		- if ($existing) {
6961		- // Update
6962		- $wpdb->update(
6963		- $meta_table,
6964		- array('meta_value' => maybe_serialize($meta_value)),
6965		- array('queue_id' => $queue_id, 'meta_key' => $meta_key),
6966		- array('%s'),
6967		- array('%s', '%s')
6968		- );
6969		- } else {
6970		- // Insert
6971		- $wpdb->insert(
6972		- $meta_table,
6973		- array(
6974		- 'queue_id' => $queue_id,
6975		- 'meta_key' => $meta_key,
6976		- 'meta_value' => maybe_serialize($meta_value)
6977		- ),
6978		- array('%s', '%s', '%s')
6979		- );
6980		- }
6981		-}
6982		-
6983		-/**
6984		- * Get queue metadata
6985		- */
6986		-private function mxchat_get_queue_meta($queue_id, $meta_key) {
6987		- global $wpdb;
6988		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
6989		-
6990		- $value = $wpdb->get_var($wpdb->prepare(
6991		- "SELECT meta_value FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
6992		- $queue_id,
6993		- $meta_key
6994		- ));
6995		-
6996		- return maybe_unserialize($value);
6997		-}
6998		-
6999		-// ========================================
7000		-// AJAX QUEUE PROCESSING HANDLERS
7001		-// ========================================
7002		-
7003		-/**
7004		- * AJAX: Get next item from queue to process
7005		- */
7006		-public function ajax_mxchat_get_next_queue_item() {
7007		- // Verify nonce and permissions
7008		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7009		-
7010		- if (!current_user_can('manage_options')) {
7011		- wp_send_json_error('Unauthorized access');
7012		- }
7013		-
7014		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7015		-
7016		- if (empty($queue_id)) {
7017		- wp_send_json_error('Missing queue ID');
7018		- }
7019		-
7020		- global $wpdb;
7021		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7022		-
7023		- // Get next pending item with retry logic for failed items
7024		- $next_item = $wpdb->get_row($wpdb->prepare(
7025		- "SELECT * FROM $table_name
7026		- WHERE queue_id = %s
7027		- AND status IN ('pending', 'failed')
7028		- AND attempts < max_attempts
7029		- ORDER BY priority ASC, id ASC
7030		- LIMIT 1",
7031		- $queue_id
7032		- ));
7033		-
7034		- if (!$next_item) {
7035		- // No more items - queue complete
7036		- wp_send_json_success(array(
7037		- 'complete' => true,
7038		- 'message' => 'Queue processing complete'
7039		- ));
7040		- }
7041		-
7042		- // Mark item as processing
7043		- $wpdb->update(
7044		- $table_name,
7045		- array(
7046		- 'status' => 'processing',
7047		- 'started_at' => current_time('mysql'),
7048		- 'attempts' => $next_item->attempts + 1
7049		- ),
7050		- array('id' => $next_item->id),
7051		- array('%s', '%s', '%d'),
7052		- array('%d')
7053		- );
7054		-
7055		- wp_send_json_success(array(
7056		- 'complete' => false,
7057		- 'item' => array(
7058		- 'id' => $next_item->id,
7059		- 'type' => $next_item->item_type,
7060		- 'data' => json_decode($next_item->item_data, true),
7061		- 'bot_id' => $next_item->bot_id,
7062		- 'attempt' => $next_item->attempts + 1
7063		- )
7064		- ));
7065		-}
7066		-
7067		-/**
7068		- * AJAX: Process a single queue item
7069		- */
7070		-public function ajax_mxchat_process_queue_item() {
7071		- // Verify nonce and permissions
7072		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7073		-
7074		- if (!current_user_can('manage_options')) {
7075		- wp_send_json_error('Unauthorized access');
7076		- }
7077		-
7078		- $item_id = isset($_POST['item_id']) ? absint($_POST['item_id']) : 0;
7079		- $item_type = isset($_POST['item_type']) ? sanitize_text_field($_POST['item_type']) : '';
7080		- $item_data = isset($_POST['item_data']) ? $_POST['item_data'] : array();
7081		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
7082		-
7083		- if (empty($item_id) || empty($item_type)) {
7084		- wp_send_json_error('Missing item data');
7085		- }
7086		-
7087		- global $wpdb;
7088		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7089		-
7090		- // Process based on item type
7091		- try {
7092		- set_time_limit(60); // Give processing 60 seconds
7093		-
7094		- $result = false;
7095		- $error_message = '';
7096		-
7097		- // Read item directly from DB to get queue_id and preserve special chars in item_data
7098		- // (POST round-trip through JS mangles characters like apostrophes in URLs)
7099		- $db_item = $wpdb->get_row($wpdb->prepare(
7100		- "SELECT queue_id, item_data FROM $table_name WHERE id = %d",
7101		- $item_id
7102		- ));
7103		- $item_queue_id = $db_item ? $db_item->queue_id : '';
7104		- if ($db_item && !empty($db_item->item_data)) {
7105		- $db_data = json_decode($db_item->item_data, true);
7106		- if (is_array($db_data)) {
7107		- $item_data = $db_data;
7108		- }
7109		- }
7110		-
7111		- switch ($item_type) {
7112		- case 'url':
7113		- $result = $this->mxchat_process_queue_url($item_data, $bot_id, $item_queue_id);
7114		- break;
7115		-
7116		- case 'pdf_page':
7117		- $result = $this->mxchat_process_queue_pdf_page($item_data, $bot_id);
7118		- break;
7119		-
7120		- default:
7121		- throw new Exception('Unknown item type: ' . $item_type);
7122		- }
7123		-
7124		- if (is_wp_error($result)) {
7125		- $error_code = $result->get_error_code();
7126		- // Content errors (empty page, sanitization) are permanent — retrying won't help
7127		- $permanent_codes = array('empty_page', 'empty_after_sanitization', 'no_api_key', 'page_not_found');
7128		- if (in_array($error_code, $permanent_codes)) {
7129		- // Mark as permanently failed — set attempts = max_attempts so it won't be retried
7130		- $current_item = $wpdb->get_row($wpdb->prepare(
7131		- "SELECT max_attempts FROM $table_name WHERE id = %d", $item_id
7132		- ));
7133		- $wpdb->update(
7134		- $table_name,
7135		- array(
7136		- 'status' => 'failed',
7137		- 'error_message' => $result->get_error_message(),
7138		- 'attempts' => $current_item ? $current_item->max_attempts : 3
7139		- ),
7140		- array('id' => $item_id),
7141		- array('%s', '%s', '%d'),
7142		- array('%d')
7143		- );
7144		- wp_send_json_error(array(
7145		- 'message' => $result->get_error_message(),
7146		- 'permanent_failure' => true,
7147		- 'item_id' => $item_id
7148		- ));
7149		- return;
7150		- }
7151		- throw new Exception($result->get_error_message());
7152		- }
7153		-
7154		- if ($result === false) {
7155		- throw new Exception('Processing returned false - item may be empty or invalid');
7156		- }
7157		-
7158		- // Mark as completed
7159		- $wpdb->update(
7160		- $table_name,
7161		- array(
7162		- 'status' => 'completed',
7163		- 'completed_at' => current_time('mysql'),
7164		- 'error_message' => null
7165		- ),
7166		- array('id' => $item_id),
7167		- array('%s', '%s', '%s'),
7168		- array('%d')
7169		- );
7170		-
7171		- wp_send_json_success(array(
7172		- 'processed' => true,
7173		- 'item_id' => $item_id,
7174		- 'message' => 'Item processed successfully'
7175		- ));
7176		-
7177		- } catch (Exception $e) {
7178		- $error_message = $e->getMessage();
7179		-
7180		- // Get current attempt count
7181		- $item = $wpdb->get_row($wpdb->prepare(
7182		- "SELECT attempts, max_attempts FROM $table_name WHERE id = %d",
7183		- $item_id
7184		- ));
7185		-
7186		- // Check if we've exhausted retries
7187		- if ($item && $item->attempts >= $item->max_attempts) {
7188		- // Permanently failed
7189		- $wpdb->update(
7190		- $table_name,
7191		- array(
7192		- 'status' => 'failed',
7193		- 'error_message' => $error_message
7194		- ),
7195		- array('id' => $item_id),
7196		- array('%s', '%s'),
7197		- array('%d')
7198		- );
7199		-
7200		- wp_send_json_error(array(
7201		- 'message' => 'Item failed after maximum attempts: ' . $error_message,
7202		- 'permanent_failure' => true,
7203		- 'item_id' => $item_id
7204		- ));
7205		- } else {
7206		- // Mark for retry
7207		- $wpdb->update(
7208		- $table_name,
7209		- array(
7210		- 'status' => 'failed',
7211		- 'error_message' => $error_message
7212		- ),
7213		- array('id' => $item_id),
7214		- array('%s', '%s'),
7215		- array('%d')
7216		- );
7217		-
7218		- wp_send_json_error(array(
7219		- 'message' => 'Item processing failed, will retry: ' . $error_message,
7220		- 'can_retry' => true,
7221		- 'item_id' => $item_id,
7222		- 'attempts' => $item ? $item->attempts : 0
7223		- ));
7224		- }
7225		- }
7226		-}
7227		-
7228		-/**
7229		- * Process a URL from the queue
7230		- */
7231		-private function mxchat_process_queue_url($item_data, $bot_id = 'default', $queue_id = '') {
7232		- $url = isset($item_data['url']) ? $item_data['url'] : '';
7233		-
7234		- if (empty($url)) {
7235		- return new WP_Error('invalid_url', 'URL is empty');
7236		- }
7237		-
7238		- // Get bot-specific API key early (needed for both paths)
7239		- $bot_options = $this->get_bot_options($bot_id);
7240		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
7241		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
7242		-
7243		- if (strpos($selected_model, 'voyage') === 0) {
7244		- $api_key = $options['voyage_api_key'] ?? '';
7245		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
7246		- $api_key = $options['gemini_api_key'] ?? '';
7247		- } else {
7248		- $api_key = $options['api_key'] ?? '';
7249		- }
7250		-
7251		- if (empty($api_key)) {
7252		- return new WP_Error('no_api_key', 'API key not configured for bot: ' . $bot_id);
7253		- }
7254		-
7255		- // Check if this is a WooCommerce product URL and WooCommerce is active
7256		- $is_product_url = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
7257		- $content_type = $is_product_url ? 'product' : 'url';
7258		-
7259		- // Try to get WooCommerce product data if it's a product URL
7260		- if ($is_product_url && class_exists('WooCommerce')) {
7261		- $product_content = $this->mxchat_extract_woocommerce_product_content($url);
7262		-
7263		- if (!empty($product_content)) {
7264		- // Successfully extracted WooCommerce product data with pricing
7265		- $result = MxChat_Utils::submit_content_to_db(
7266		- $product_content,
7267		- $url,
7268		- $api_key,
7269		- null,
7270		- $bot_id,
7271		- 'product'
7272		- );
7273		- return $result;
7274		- }
7275		- // If WooCommerce extraction failed, fall through to HTML extraction
7276		- }
7277		-
7278		- // Fetch URL content (fallback for non-products or when WooCommerce extraction fails)
7279		- $is_likely_pdf = (strtolower(pathinfo(parse_url($url, PHP_URL_PATH) ?: '', PATHINFO_EXTENSION)) === 'pdf');
7280		- $response = wp_remote_get($url, array(
7281		- 'timeout' => $is_likely_pdf ? 120 : 30,
7282		- 'redirection' => 5,
7283		- 'user-agent' => 'MxChat/1.0'
7284		- ));
7285		-
7286		- if (is_wp_error($response)) {
7287		- return $response;
7288		- }
7289		-
7290		- $response_code = wp_remote_retrieve_response_code($response);
7291		- if ($response_code !== 200) {
7292		- return new WP_Error('http_error', 'HTTP ' . $response_code . ' error for: ' . $url);
7293		- }
7294		-
7295		- // Check if URL is a PDF — expand into per-page queue items using the standard PDF pipeline
7296		- if ($this->mxchat_is_pdf_url($url, $response)) {
7297		- return $this->mxchat_expand_pdf_to_queue($url, $response, $bot_id, $queue_id);
7298		- }
7299		-
7300		- $html = wp_remote_retrieve_body($response);
7301		-
7302		- if (empty($html)) {
7303		- return new WP_Error('empty_response', 'Empty response body');
7304		- }
7305		-
7306		- // Extract and sanitize content
7307		- $content = $this->mxchat_extract_main_content($html);
7308		- $sanitized = $this->mxchat_sanitize_content_for_api($content);
7309		-
7310		- if (empty($sanitized)) {
7311		- // Not an error - just no content found (maybe a redirect or empty page)
7312		- return false;
7313		- }
7314		-
7315		- // Submit to database with content_type
7316		- $result = MxChat_Utils::submit_content_to_db(
7317		- $sanitized,
7318		- $url,
7319		- $api_key,
7320		- null,
7321		- $bot_id,
7322		- $content_type
7323		- );
7324		-
7325		- return $result;
7326		-}
7327		-
7328		-/**
7329		- * Expand a PDF URL into per-page queue items using the standard PDF pipeline.
7330		- * Called when a sitemap URL turns out to be a PDF — downloads, parses page count,
7331		- * and adds pdf_page items to the same queue so they process with full progress tracking.
7332		- */
7333		-private function mxchat_expand_pdf_to_queue($pdf_url, $response, $bot_id = 'default', $queue_id = '') {
7334		- set_time_limit(120); // PDFs need extra time for download + parsing
7335		-
7336		- $upload_dir = wp_upload_dir();
7337		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
7338		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
7339		-
7340		- $response_body = wp_remote_retrieve_body($response);
7341		- if (empty($response_body)) {
7342		- return new WP_Error('empty_pdf', 'Empty PDF response for: ' . $pdf_url);
7343		- }
7344		-
7345		- if (!wp_mkdir_p(dirname($pdf_path))) {
7346		- return new WP_Error('dir_error', 'Failed to create upload directory');
7347		- }
7348		-
7349		- file_put_contents($pdf_path, $response_body);
7350		-
7351		- if (!file_exists($pdf_path)) {
7352		- return new WP_Error('save_error', 'Failed to save PDF file');
7353		- }
7354		-
7355		- try {
7356		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
7357		-
7358		- if ($total_pages === false || $total_pages < 1) {
7359		- wp_delete_file($pdf_path);
7360		- return new WP_Error('no_pages', 'PDF has no pages: ' . $pdf_url);
7361		- }
7362		-
7363		- // Build per-page items identical to mxchat_handle_pdf_for_knowledge_base
7364		- $pages = array();
7365		- for ($i = 1; $i <= $total_pages; $i++) {
7366		- $pages[] = array(
7367		- 'pdf_path' => $pdf_path,
7368		- 'pdf_url' => $pdf_url,
7369		- 'page_number' => $i,
7370		- 'total_pages' => $total_pages
7371		- );
7372		- }
7373		-
7374		- // Add pdf_page items to the SAME queue so the JS picks them up automatically
7375		- if (!empty($queue_id)) {
7376		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
7377		- } else {
7378		- // Fallback: create a new PDF queue (shouldn't happen in sitemap flow)
7379		- $new_queue_id = 'pdf_' . md5($pdf_url . time());
7380		- $queued_count = $this->mxchat_add_to_queue($new_queue_id, 'pdf_page', $pages, $bot_id);
7381		- $this->mxchat_set_queue_meta($new_queue_id, 'source_url', $pdf_url);
7382		- $this->mxchat_set_queue_meta($new_queue_id, 'queue_type', 'pdf');
7383		- $this->mxchat_set_queue_meta($new_queue_id, 'total_items', $total_pages);
7384		- $this->mxchat_set_queue_meta($new_queue_id, 'bot_id', $bot_id);
7385		- $this->mxchat_set_queue_meta($new_queue_id, 'pdf_path', $pdf_path);
7386		- $this->mxchat_set_queue_meta($new_queue_id, 'created_at', current_time('mysql'));
7387		- }
7388		-
7389		- if ($queued_count === 0) {
7390		- wp_delete_file($pdf_path);
7391		- return new WP_Error('queue_error', 'Failed to add PDF pages to queue');
7392		- }
7393		-
7394		- // Return true so the original URL item is marked complete
7395		- // The new pdf_page items will be processed in subsequent batches
7396		- return true;
7397		-
7398		- } catch (Exception $e) {
7399		- if (file_exists($pdf_path)) {
7400		- wp_delete_file($pdf_path);
7401		- }
7402		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
7403		- }
7404		-}
7405		-
7406		-/**
7407		- * Legacy: Process a PDF URL inline during sitemap queue processing.
7408		- * @deprecated Use mxchat_expand_pdf_to_queue instead — kept for reference only.
7409		- */
7410		-private function mxchat_process_pdf_url_inline($pdf_url, $response, $api_key, $bot_id = 'default') {
7411		- set_time_limit(120); // PDFs need more time — downloading + parsing all pages
7412		-
7413		- $upload_dir = wp_upload_dir();
7414		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
7415		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
7416		-
7417		- $response_body = wp_remote_retrieve_body($response);
7418		- if (empty($response_body)) {
7419		- return new WP_Error('empty_pdf', 'Empty PDF response');
7420		- }
7421		-
7422		- if (!wp_mkdir_p(dirname($pdf_path))) {
7423		- return new WP_Error('dir_error', 'Failed to create upload directory');
7424		- }
7425		-
7426		- file_put_contents($pdf_path, $response_body);
7427		-
7428		- if (!file_exists($pdf_path)) {
7429		- return new WP_Error('save_error', 'Failed to save PDF file');
7430		- }
7431		-
7432		- try {
7433		- mxchat_load_pdf_parser();
7434		- $parser = new \Smalot\PdfParser\Parser();
7435		- $pdf = $parser->parseFile($pdf_path);
7436		- $pages = $pdf->getPages();
7437		- $total_pages = count($pages);
7438		-
7439		- if ($total_pages < 1) {
7440		- wp_delete_file($pdf_path);
7441		- return new WP_Error('no_pages', 'PDF has no pages');
7442		- }
7443		-
7444		- $processed = 0;
7445		- $skipped_pages = array();
7446		-
7447		- for ($i = 0; $i < $total_pages; $i++) {
7448		- $page_num = $i + 1;
7449		- $text = $pages[$i]->getText();
7450		- if (empty($text)) {
7451		- $skipped_pages[] = 'Page ' . $page_num . ': No text could be extracted — page may contain only images, links, or non-standard encoding';
7452		- continue;
7453		- }
7454		-
7455		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
7456		- if (empty($sanitized)) {
7457		- $skipped_pages[] = 'Page ' . $page_num . ': Text was extracted but contained only special characters, control codes, or unsupported content';
7458		- continue;
7459		- }
7460		-
7461		- $metadata = array(
7462		- 'document_type' => 'pdf',
7463		- 'total_pages' => $total_pages,
7464		- 'current_page' => $page_num,
7465		- 'source_url' => $pdf_url,
7466		- );
7467		-
7468		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
7469		- $page_url = esc_url($pdf_url . '#page=' . $page_num);
7470		-
7471		- MxChat_Utils::submit_content_to_db(
7472		- $content_with_metadata,
7473		- $page_url,
7474		- $api_key,
7475		- null,
7476		- $bot_id,
7477		- 'pdf'
7478		- );
7479		-
7480		- $processed++;
7481		- }
7482		-
7483		- // Clean up the temp PDF file
7484		- wp_delete_file($pdf_path);
7485		-
7486		- if (!empty($skipped_pages)) {
7487		- error_log('MxChat PDF: Skipped ' . count($skipped_pages) . ' of ' . $total_pages . ' pages: ' . implode('; ', $skipped_pages));
7488		- }
7489		-
7490		- return $processed > 0 ? true : false;
7491		-
7492		- } catch (Exception $e) {
7493		- if (file_exists($pdf_path)) {
7494		- wp_delete_file($pdf_path);
7495		- }
7496		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
7497		- }
7498		-}
7499		-
7500		-/**
7501		- * Extract WooCommerce product content including pricing
7502		- *
7503		- * @param string $url The product URL
7504		- * @return string|false Product content with pricing, or false if not found
7505		- */
7506		-private function mxchat_extract_woocommerce_product_content($url) {
7507		- // Try to get product ID from URL
7508		- $product_id = url_to_postid($url);
7509		-
7510		- // If url_to_postid fails, try to extract from URL pattern
7511		- if (!$product_id) {
7512		- $product_slug = '';
7513		-
7514		- // Handle pretty permalinks: /product/product-name/
7515		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
7516		- $product_slug = $matches[1];
7517		- }
7518		-
7519		- if (!empty($product_slug)) {
7520		- $product_post = get_page_by_path($product_slug, OBJECT, 'product');
7521		- if ($product_post) {
7522		- $product_id = $product_post->ID;
7523		- }
7524		- }
7525		- }
7526		-
7527		- if (!$product_id) {
7528		- return false;
7529		- }
7530		-
7531		- // Get WooCommerce product object
7532		- $product = wc_get_product($product_id);
7533		-
7534		- if (!$product) {
7535		- return false;
7536		- }
7537		-
7538		- // Build product content with pricing (similar to mxchat_store_product_embedding)
7539		- $title = $product->get_name();
7540		- $description = $product->get_description();
7541		- $short_description = $product->get_short_description();
7542		- $sku = $product->get_sku();
7543		-
7544		- // Get pricing information
7545		- $regular_price = $product->get_regular_price();
7546		- $sale_price = $product->get_sale_price();
7547		- $price = $product->get_price(); // Current active price
7548		-
7549		- // Get currency symbol
7550		- $currency_symbol = get_woocommerce_currency_symbol();
7551		-
7552		- // Format content
7553		- $content = $title . "\n\n";
7554		-
7555		- if (!empty($short_description)) {
7556		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
7557		- }
7558		-
7559		- if (!empty($description)) {
7560		- $content .= wp_strip_all_tags($description) . "\n\n";
7561		- }
7562		-
7563		- // Add pricing information
7564		- if (!empty($regular_price)) {
7565		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
7566		- } elseif (!empty($price)) {
7567		- $content .= "Price: " . $currency_symbol . $price . "\n";
7568		- }
7569		-
7570		- if (!empty($sale_price) && $sale_price !== $regular_price) {
7571		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
7572		- }
7573		-
7574		- // Handle variable products - show price range
7575		- if ($product->is_type('variable')) {
7576		- $min_price = $product->get_variation_price('min');
7577		- $max_price = $product->get_variation_price('max');
7578		- if ($min_price !== $max_price) {
7579		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
7580		- }
7581		- }
7582		-
7583		- if (!empty($sku)) {
7584		- $content .= "SKU: " . $sku . "\n";
7585		- }
7586		-
7587		- // Get product categories
7588		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
7589		- if (!empty($categories) && !is_wp_error($categories)) {
7590		- $content .= "Categories: " . implode(', ', $categories) . "\n";
7591		- }
7592		-
7593		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
7594		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
7595		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
7596		- foreach ($custom_tabs as $tab) {
7597		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
7598		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
7599		-
7600		- if (!empty($tab_title) && !empty($tab_content)) {
7601		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
7602		- }
7603		- }
7604		- }
7605		-
7606		- // Also check for reusable/saved tabs applied to this product
7607		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
7608		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
7609		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
7610		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
7611		- foreach ($applied_saved_tabs as $saved_tab_id) {
7612		- if (isset($saved_tabs[$saved_tab_id])) {
7613		- $tab = $saved_tabs[$saved_tab_id];
7614		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
7615		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
7616		-
7617		- if (!empty($tab_title) && !empty($tab_content)) {
7618		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
7619		- }
7620		- }
7621		- }
7622		- }
7623		- }
7624		-
7625		- return $this->mxchat_sanitize_content_for_api($content);
7626		-}
7627		-
7628		-/**
7629		- * Process a PDF page from the queue
7630		- */
7631		-private function mxchat_process_queue_pdf_page($item_data, $bot_id = 'default') {
7632		- $pdf_path = isset($item_data['pdf_path']) ? $item_data['pdf_path'] : '';
7633		- $pdf_url = isset($item_data['pdf_url']) ? $item_data['pdf_url'] : '';
7634		- $page_number = isset($item_data['page_number']) ? absint($item_data['page_number']) : 0;
7635		- $total_pages = isset($item_data['total_pages']) ? absint($item_data['total_pages']) : 0;
7636		-
7637		- if (empty($pdf_path) || !file_exists($pdf_path)) {
7638		- return new WP_Error('pdf_not_found', 'PDF file not found: ' . $pdf_path);
7639		- }
7640		-
7641		- if ($page_number < 1) {
7642		- return new WP_Error('invalid_page', 'Invalid page number');
7643		- }
7644		-
7645		- try {
7646		- mxchat_load_pdf_parser();
7647		- $parser = new \Smalot\PdfParser\Parser();
7648		- $pdf = $parser->parseFile($pdf_path);
7649		- $pages = $pdf->getPages();
7650		-
7651		- if (!isset($pages[$page_number - 1])) {
7652		- return new WP_Error('page_not_found', 'Page ' . $page_number . ' not found in PDF');
7653		- }
7654		-
7655		- $text = $pages[$page_number - 1]->getText();
7656		-
7657		- if (empty($text)) {
7658		- return new WP_Error('empty_page', 'Page ' . $page_number . ': No text could be extracted — page may contain only images, links, or non-standard encoding');
7659		- }
7660		-
7661		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
7662		-
7663		- if (empty($sanitized)) {
7664		- return new WP_Error('empty_after_sanitization', 'Page ' . $page_number . ': Text was extracted but contained only special characters, control codes, or unsupported content that was removed during cleanup');
7665		- }
7666		-
7667		- // Create metadata
7668		- $metadata = array(
7669		- 'document_type' => 'pdf',
7670		- 'total_pages' => $total_pages,
7671		- 'current_page' => $page_number,
7672		- 'source_url' => $pdf_url
7673		- );
7674		-
7675		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
7676		- $page_url = esc_url($pdf_url . "#page=" . $page_number);
7677		-
7678		- // Get bot-specific API key
7679		- $bot_options = $this->get_bot_options($bot_id);
7680		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
7681		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
7682		-
7683		- if (strpos($selected_model, 'voyage') === 0) {
7684		- $api_key = $options['voyage_api_key'] ?? '';
7685		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
7686		- $api_key = $options['gemini_api_key'] ?? '';
7687		- } else {
7688		- $api_key = $options['api_key'] ?? '';
7689		- }
7690		-
7691		- if (empty($api_key)) {
7692		- return new WP_Error('no_api_key', 'API key not configured for bot: ' . $bot_id);
7693		- }
7694		-
7695		- // Submit to database - UPDATED 2.5.6: Added content_type 'pdf'
7696		- $result = MxChat_Utils::submit_content_to_db(
7697		- $content_with_metadata,
7698		- $page_url,
7699		- $api_key,
7700		- null,
7701		- $bot_id,
7702		- 'pdf'
7703		- );
7704		-
7705		- return $result;
7706		-
7707		- } catch (Exception $e) {
7708		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
7709		- }
7710		-}
7711		-
7712		-/**
7713		- * AJAX: Get queue processing status
7714		- */
7715		-public function ajax_mxchat_get_queue_status() {
7716		- // Verify nonce and permissions
7717		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7718		-
7719		- if (!current_user_can('manage_options')) {
7720		- wp_send_json_error('Unauthorized access');
7721		- }
7722		-
7723		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7724		-
7725		- if (empty($queue_id)) {
7726		- wp_send_json_error('Missing queue ID');
7727		- }
7728		-
7729		- global $wpdb;
7730		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7731		-
7732		- // Get counts by status
7733		- $counts = $wpdb->get_results($wpdb->prepare(
7734		- "SELECT status, COUNT(*) as count
7735		- FROM $table_name
7736		- WHERE queue_id = %s
7737		- GROUP BY status",
7738		- $queue_id
7739		- ), OBJECT_K);
7740		-
7741		- $total = 0;
7742		- $completed = 0;
7743		- $failed = 0;
7744		- $processing = 0;
7745		- $pending = 0;
7746		-
7747		- foreach ($counts as $status => $data) {
7748		- $count = absint($data->count);
7749		- $total += $count;
7750		-
7751		- switch ($status) {
7752		- case 'completed':
7753		- $completed = $count;
7754		- break;
7755		- case 'failed':
7756		- $failed = $count;
7757		- break;
7758		- case 'processing':
7759		- $processing = $count;
7760		- break;
7761		- case 'pending':
7762		- $pending = $count;
7763		- break;
7764		- }
7765		- }
7766		-
7767		- // Calculate percentage
7768		- $percentage = $total > 0 ? round((($completed + $failed) / $total) * 100) : 0;
7769		-
7770		- // Get failed items details (include all failed items, not just those that exhausted retries)
7771		- $failed_items = array();
7772		- if ($failed > 0) {
7773		- $failed_items = $wpdb->get_results($wpdb->prepare(
7774		- "SELECT item_type, item_data, error_message, attempts
7775		- FROM $table_name
7776		- WHERE queue_id = %s
7777		- AND status = 'failed'
7778		- ORDER BY id DESC
7779		- LIMIT 50",
7780		- $queue_id
7781		- ));
7782		- }
7783		-
7784		- // Get queue metadata
7785		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
7786		- $queue_type = $this->mxchat_get_queue_meta($queue_id, 'queue_type');
7787		-
7788		- // Determine if queue is complete
7789		- $is_complete = ($pending === 0 && $processing === 0);
7790		-
7791		- wp_send_json_success(array(
7792		- 'queue_id' => $queue_id,
7793		- 'queue_type' => $queue_type,
7794		- 'source_url' => $source_url,
7795		- 'total' => $total,
7796		- 'completed' => $completed,
7797		- 'failed' => $failed,
7798		- 'processing' => $processing,
7799		- 'pending' => $pending,
7800		- 'percentage' => $percentage,
7801		- 'is_complete' => $is_complete,
7802		- 'failed_items' => $failed_items,
7803		- 'status' => $is_complete ? 'complete' : 'processing'
7804		- ));
7805		-}
7806		-
7807		-/**
7808		- * AJAX: Clear completed queue
7809		- */
7810		-public function ajax_mxchat_clear_queue() {
7811		- // Verify nonce and permissions
7812		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7813		-
7814		- if (!current_user_can('manage_options')) {
7815		- wp_send_json_error('Unauthorized access');
7816		- }
7817		-
7818		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7819		-
7820		- if (empty($queue_id)) {
7821		- wp_send_json_error('Missing queue ID');
7822		- }
7823		-
7824		- global $wpdb;
7825		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7826		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7827		-
7828		- // Delete queue items
7829		- $wpdb->delete(
7830		- $table_name,
7831		- array('queue_id' => $queue_id),
7832		- array('%s')
7833		- );
7834		-
7835		- // Delete queue metadata
7836		- $wpdb->delete(
7837		- $meta_table,
7838		- array('queue_id' => $queue_id),
7839		- array('%s')
7840		- );
7841		-
7842		- wp_send_json_success(array(
7843		- 'message' => 'Queue cleared successfully'
7844		- ));
7845		-}
7846		-
7847		-/**
7848		- * AJAX: Retry failed items in queue
7849		- */
7850		-public function ajax_mxchat_retry_failed() {
7851		- // Verify nonce and permissions
7852		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7853		-
7854		- if (!current_user_can('manage_options')) {
7855		- wp_send_json_error('Unauthorized access');
7856		- }
7857		-
7858		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7859		-
7860		- if (empty($queue_id)) {
7861		- wp_send_json_error('Missing queue ID');
7862		- }
7863		-
7864		- global $wpdb;
7865		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7866		-
7867		- // Reset failed items to pending and reset attempt count
7868		- $updated = $wpdb->update(
7869		- $table_name,
7870		- array(
7871		- 'status' => 'pending',
7872		- 'attempts' => 0,
7873		- 'error_message' => null
7874		- ),
7875		- array(
7876		- 'queue_id' => $queue_id,
7877		- 'status' => 'failed'
7878		- ),
7879		- array('%s', '%d', '%s'),
7880		- array('%s', '%s')
7881		- );
7882		-
7883		- wp_send_json_success(array(
7884		- 'message' => 'Reset ' . $updated . ' failed items for retry',
7885		- 'reset_count' => $updated
7886		- ));
7887		-}
7888		-
7889		-
7890		-public function ajax_mxchat_mark_queue_complete() {
7891		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7892		-
7893		- if (!current_user_can('manage_options')) {
7894		- wp_send_json_error('Unauthorized access');
7895		- }
7896		-
7897		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7898		-
7899		- if (empty($queue_id)) {
7900		- wp_send_json_error('Missing queue ID');
7901		- }
7902		-
7903		- // Clear active queue transients
7904		- if (strpos($queue_id, 'sitemap_') === 0) {
7905		- delete_transient('mxchat_active_queue_sitemap');
7906		- } else if (strpos($queue_id, 'pdf_') === 0) {
7907		- delete_transient('mxchat_active_queue_pdf');
7908		- }
7909		-
7910		- wp_send_json_success(array('message' => 'Queue marked as complete'));
7911		-}
7912		-
7913	3346
7914	3347	// ========================================
7915	3348	// STATIC ACCESS METHODS
7916	3349	// ========================================