← All changes | admin/class-knowledge-manager.php +1637 -6722 3.2.14 → 2.2.7 View file →

		@@ -15,74 +15,51 @@
15	15
16	16	/**
17	17	* Constructor - Register hooks for content processing
18	18	*/
19		-public function __construct() {
20		- $this->options = get_option('mxchat_options', array());
21		- $this->mxchat_init_hooks();
	19	+ public function __construct() {
	20	+ $this->options = get_option('mxchat_options', array());
	21	+ $this->mxchat_init_hooks();
	22	+ }
22	23
23		- $this->mxchat_init_role_hooks();
24		-}
25		-
26		-/**
27		- * Initialize WordPress hooks for content processing
28		- *
29		- */
30		-private function mxchat_init_hooks() {
31		- // Admin post handlers for form submissions
32		- add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
33		- add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
34		- add_action('admin_post_mxchat_submit_pdf_file', array($this, 'mxchat_handle_pdf_file_submission'));
35		- add_action('admin_post_mxchat_submit_youtube', array($this, 'mxchat_handle_youtube_submission'));
36		- add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
37		-
38		- // AJAX handlers for real-time processing and status updates
39		- add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
40		- add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status'));
41		- add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
42		- add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
43		- add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
44		- add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
45		- add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
46		- add_action('wp_ajax_mxchat_delete_chunks_by_url', array($this, 'ajax_mxchat_delete_chunks_by_url'));
47		- add_action('wp_ajax_mxchat_delete_wordpress_prompt', array($this, 'ajax_mxchat_delete_wordpress_prompt'));
48		- add_action('wp_ajax_mxchat_bulk_delete_knowledge', array($this, 'ajax_mxchat_bulk_delete_knowledge'));
49		- add_action('wp_ajax_mxchat_update_role_restriction', array($this, 'ajax_mxchat_update_role_restriction'));
50		-
51		- // Queue-based processing AJAX handlers
52		- add_action('wp_ajax_mxchat_get_next_queue_item', array($this, 'ajax_mxchat_get_next_queue_item'));
53		- add_action('wp_ajax_mxchat_process_queue_item', array($this, 'ajax_mxchat_process_queue_item'));
54		- add_action('wp_ajax_mxchat_get_queue_status', array($this, 'ajax_mxchat_get_queue_status'));
55		- add_action('wp_ajax_mxchat_clear_queue', array($this, 'ajax_mxchat_clear_queue'));
56		- add_action('wp_ajax_mxchat_retry_failed', array($this, 'ajax_mxchat_retry_failed'));
57		- add_action('wp_ajax_mxchat_get_recent_entries', array($this, 'ajax_mxchat_get_recent_entries'));
58		- add_action('wp_ajax_mxchat_detect_sitemaps', array($this, 'ajax_mxchat_detect_sitemaps'));
59		- add_action('wp_ajax_mxchat_refresh_pinecone_entries', array($this, 'ajax_mxchat_refresh_pinecone_entries'));
60		- add_action('wp_ajax_mxchat_paginate_entries', array($this, 'ajax_mxchat_paginate_entries'));
61		- add_action('wp_ajax_mxchat_get_entry_content', array($this, 'ajax_mxchat_get_entry_content'));
62		- add_action('wp_ajax_mxchat_save_entry_content', array($this, 'ajax_mxchat_save_entry_content'));
63		- add_action('wp_ajax_mxchat_inspect_entry', array($this, 'ajax_mxchat_inspect_entry'));
	24	+ /**
	25	+ * Initialize WordPress hooks for content processing
	26	+ */
	27	+ private function mxchat_init_hooks() {
	28	+ // Admin post handlers for form submissions
	29	+ add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
	30	+ add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
	31	+ add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
	32	+
	33	+ // AJAX handlers for real-time processing and status updates
	34	+ add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
	35	+ add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status')); // NEW
	36	+ add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
	37	+ add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
	38	+ add_action('wp_ajax_mxchat_manual_batch_process', array($this, 'ajax_manual_batch_process'));
	39	+ add_action('mxchat_delete_content', array($this, 'mxchat_delete_from_pinecone_by_url'), 10, 1);
64	40
65		- // WordPress post management hooks
66		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
67		- add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
68		- add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
69		- add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
70		-
71		- // ACF hook - fires AFTER ACF fields are saved, ensuring ACF data is available
72		- // Priority 20 to run after ACF's own save (which runs at priority 10)
73		- add_action('acf/save_post', array($this, 'mxchat_handle_acf_save'), 20);
74		-
75		- add_action('wp_ajax_mxchat_mark_queue_complete', array($this, 'ajax_mxchat_mark_queue_complete'));
76		-
77		- // WooCommerce product hooks (if WooCommerce is active)
78		- if (class_exists('WooCommerce')) {
79		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
80		- add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
81		- add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
82		- add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
	41	+
	42	+ // Cron handlers for background processing
	43	+ add_action('mxchat_process_sitemap_urls', array($this, 'mxchat_process_sitemap_urls_cron'), 10, 5);
	44	+ add_action('mxchat_process_pdf_pages', array($this, 'mxchat_process_pdf_pages_cron'), 10, 5);
	45	+
	46	+ // WordPress post management hooks
	47	+ add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
	48	+ add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
	49	+ add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
	50	+ add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
	51	+ add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
	52	+ add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
	53	+
	54	+ // WooCommerce product hooks (if WooCommerce is active)
	55	+ if (class_exists('WooCommerce')) {
	56	+ add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
	57	+ add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
	58	+ add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
	59	+ }
	60	+
83	61	}
84		-}
85	62
86	63	/**
87	64	* Get current options (refreshed)
88	65	*/
		@@ -92,9 +69,315 @@
92	69	}
93	70	return $this->options;
94	71	}
95	72
	73	+
	74	+ /**
	75	+ * Handle manual batch processing via AJAX
	76	+ */
	77	+public function ajax_manual_batch_process() {
	78	+ try {
	79	+ // Verify nonce and permissions
	80	+ check_ajax_referer('mxchat_status_nonce', 'nonce');
	81	+
	82	+ if (!current_user_can('manage_options')) {
	83	+ wp_send_json_error('Unauthorized access');
	84	+ }
	85	+
	86	+ $process_type = sanitize_text_field($_POST['process_type'] ?? '');
	87	+ $url = sanitize_text_field($_POST['url'] ?? '');
	88	+
	89	+ if (empty($process_type) || empty($url)) {
	90	+ wp_send_json_error('Missing required parameters');
	91	+ }
	92	+
	93	+ $processed = 0;
	94	+
	95	+ if ($process_type === 'pdf') {
	96	+ $processed = $this->mxchat_manual_process_pdf_batch($url);
	97	+ } elseif ($process_type === 'sitemap') {
	98	+ $processed = $this->mxchat_manual_process_sitemap_batch($url);
	99	+ }
	100	+
	101	+ if ($processed > 0) {
	102	+ wp_send_json_success(array(
	103	+ 'message' => "Processed {$processed} items successfully",
	104	+ 'processed' => $processed
	105	+ ));
	106	+ } else {
	107	+ wp_send_json_error('No items were processed');
	108	+ }
	109	+
	110	+ } catch (Exception $e) {
	111	+ //error_log('Manual batch process error: ' . $e->getMessage());
	112	+ wp_send_json_error('Processing failed: ' . $e->getMessage());
	113	+ }
	114	+}
96	115
	116	+/**
	117	+ * Process a small PDF batch manually - DIRECT PROCESSING
	118	+ */
	119	+private function mxchat_manual_process_pdf_batch($pdf_url) {
	120	+ try {
	121	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	122	+ $status = get_transient($status_key);
	123	+
	124	+ if (!$status || $status['status'] !== 'processing') {
	125	+ //error_log('Manual PDF: No processing status found');
	126	+ return 0;
	127	+ }
	128	+
	129	+ //error_log('Manual PDF: Starting direct processing for ' . $pdf_url);
	130	+
	131	+ // Get current progress
	132	+ $current_page = $status['processed_pages'] ?? 0;
	133	+ $total_pages = $status['total_pages'] ?? 0;
	134	+
	135	+ if ($current_page >= $total_pages) {
	136	+ //error_log('Manual PDF: Already completed');
	137	+ return 0;
	138	+ }
	139	+
	140	+ // Try to download the PDF again for processing
	141	+ $response = wp_remote_get($pdf_url, array('timeout' => 30));
	142	+
	143	+ if (is_wp_error($response)) {
	144	+ //error_log('Manual PDF: Failed to download PDF: ' . $response->get_error_message());
	145	+ return 0;
	146	+ }
	147	+
	148	+ $pdf_content = wp_remote_retrieve_body($response);
	149	+ if (empty($pdf_content)) {
	150	+ //error_log('Manual PDF: Empty PDF content');
	151	+ return 0;
	152	+ }
	153	+
	154	+ // Save PDF temporarily
	155	+ $upload_dir = wp_upload_dir();
	156	+ $temp_pdf_path = trailingslashit($upload_dir['path']) . 'temp_manual_' . time() . '.pdf';
	157	+ file_put_contents($temp_pdf_path, $pdf_content);
	158	+
	159	+ // Process 2 pages directly
	160	+ $processed = $this->mxchat_process_pdf_pages_direct($temp_pdf_path, $pdf_url, $current_page, 5);
	161	+
	162	+ // Clean up temp file
	163	+ if (file_exists($temp_pdf_path)) {
	164	+ wp_delete_file($temp_pdf_path);
	165	+ }
	166	+
	167	+ //error_log('Manual PDF: Processed ' . $processed . ' pages');
	168	+ return $processed;
	169	+
	170	+ } catch (Exception $e) {
	171	+ //error_log('Manual PDF batch error: ' . $e->getMessage());
	172	+ return 0;
	173	+ }
	174	+}
	175	+
	176	+/**
	177	+ * Process PDF pages directly without cron
	178	+ */
	179	+private function mxchat_process_pdf_pages_direct($pdf_path, $pdf_url, $start_page, $batch_size) {
	180	+ try {
	181	+ if (!file_exists($pdf_path)) {
	182	+ //error_log('Direct PDF: File not found at ' . $pdf_path);
	183	+ return 0;
	184	+ }
	185	+
	186	+ $parser = new \Smalot\PdfParser\Parser();
	187	+ $pdf = $parser->parseFile($pdf_path);
	188	+ $pages = $pdf->getPages();
	189	+
	190	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	191	+ $status = get_transient($status_key);
	192	+
	193	+ if (!$status) {
	194	+ return 0;
	195	+ }
	196	+
	197	+ $options = get_option('mxchat_options');
	198	+ $api_key = $options['api_key'] ?? '';
	199	+
	200	+ if (empty($api_key)) {
	201	+ //error_log('Direct PDF: No API key');
	202	+ return 0;
	203	+ }
	204	+
	205	+ $processed = 0;
	206	+ $end_page = min($start_page + $batch_size, count($pages));
	207	+
	208	+ for ($i = $start_page; $i < $end_page; $i++) {
	209	+ try {
	210	+ $page_number = $i + 1;
	211	+ $text = $pages[$i]->getText();
	212	+
	213	+ if (empty($text)) {
	214	+ //error_log('Direct PDF: Empty text on page ' . $page_number);
	215	+ continue;
	216	+ }
	217	+
	218	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
	219	+ if (empty($sanitized_content)) {
	220	+ //error_log('Direct PDF: No content after sanitization on page ' . $page_number);
	221	+ continue;
	222	+ }
	223	+
	224	+ // Generate embedding
	225	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	226	+ if (is_string($embedding_vector)) {
	227	+ //error_log('Direct PDF: Embedding failed on page ' . $page_number . ': ' . $embedding_vector);
	228	+ continue;
	229	+ }
	230	+
	231	+ // Create metadata
	232	+ $metadata = array(
	233	+ 'document_type' => 'pdf',
	234	+ 'total_pages' => count($pages),
	235	+ 'current_page' => $page_number,
	236	+ 'source_url' => $pdf_url
	237	+ );
	238	+
	239	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	240	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
	241	+
	242	+ // Store in database
	243	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $api_key);
	244	+
	245	+ if (is_wp_error($db_result)) {
	246	+ //error_log('Direct PDF: DB error on page ' . $page_number . ': ' . $db_result->get_error_message());
	247	+ continue;
	248	+ }
	249	+
	250	+ $processed++;
	251	+ //error_log('Direct PDF: Successfully processed page ' . $page_number);
	252	+
	253	+ // Update status
	254	+ $status['processed_pages'] = $i + 1;
	255	+ $status['last_update'] = time();
	256	+ $status['percentage'] = round(($status['processed_pages'] / $status['total_pages']) * 100);
	257	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	258	+
	259	+ } catch (Exception $e) {
	260	+ //error_log('Direct PDF: Error processing page ' . ($i + 1) . ': ' . $e->getMessage());
	261	+ continue;
	262	+ }
	263	+ }
	264	+
	265	+ // Check if completed
	266	+ if ($status['processed_pages'] >= $status['total_pages']) {
	267	+ $status['status'] = 'complete';
	268	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	269	+ //error_log('Direct PDF: Processing completed');
	270	+ }
	271	+
	272	+ return $processed;
	273	+
	274	+ } catch (Exception $e) {
	275	+ //error_log('Direct PDF processing error: ' . $e->getMessage());
	276	+ return 0;
	277	+ }
	278	+}
	279	+
	280	+/**
	281	+ * Process a small sitemap batch manually - DIRECT PROCESSING
	282	+ */
	283	+private function mxchat_manual_process_sitemap_batch($sitemap_url) {
	284	+ try {
	285	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	286	+ $status = get_transient($status_key);
	287	+
	288	+ if (!$status || $status['status'] !== 'processing') {
	289	+ return 0;
	290	+ }
	291	+
	292	+ //error_log('Manual Sitemap: Starting direct processing for ' . $sitemap_url);
	293	+
	294	+ // Re-fetch the sitemap to get URLs
	295	+ $response = wp_remote_get($sitemap_url, array('timeout' => 30));
	296	+ if (is_wp_error($response)) {
	297	+ //error_log('Manual Sitemap: Failed to fetch sitemap');
	298	+ return 0;
	299	+ }
	300	+
	301	+ $sitemap_content = wp_remote_retrieve_body($response);
	302	+ $xml = simplexml_load_string($sitemap_content);
	303	+
	304	+ if (!$xml) {
	305	+ //error_log('Manual Sitemap: Invalid XML');
	306	+ return 0;
	307	+ }
	308	+
	309	+ $urls = array();
	310	+ foreach ($xml->url as $url_element) {
	311	+ $urls[] = (string)$url_element->loc;
	312	+ }
	313	+
	314	+ $current_processed = $status['processed_urls'] ?? 0;
	315	+ $batch_size = 5;
	316	+ $processed = 0;
	317	+
	318	+ // Process next 2 URLs
	319	+ for ($i = $current_processed; $i < min($current_processed + $batch_size, count($urls)); $i++) {
	320	+ $url = $urls[$i];
	321	+
	322	+ if ($this->mxchat_process_single_url_direct($url)) {
	323	+ $processed++;
	324	+ }
	325	+
	326	+ // Update status
	327	+ $status['processed_urls'] = $i + 1;
	328	+ $status['last_update'] = time();
	329	+ $status['percentage'] = round(($status['processed_urls'] / $status['total_urls']) * 100);
	330	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	331	+ }
	332	+
	333	+ // Check if completed
	334	+ if ($status['processed_urls'] >= $status['total_urls']) {
	335	+ $status['status'] = 'complete';
	336	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	337	+ }
	338	+
	339	+ //error_log('Manual Sitemap: Processed ' . $processed . ' URLs');
	340	+ return $processed;
	341	+
	342	+ } catch (Exception $e) {
	343	+ //error_log('Manual sitemap batch error: ' . $e->getMessage());
	344	+ return 0;
	345	+ }
	346	+}
	347	+
	348	+/**
	349	+ * Process a single URL directly
	350	+ */
	351	+private function mxchat_process_single_url_direct($url) {
	352	+ try {
	353	+ $response = wp_remote_get($url, array('timeout' => 30));
	354	+ if (is_wp_error($response)) {
	355	+ return false;
	356	+ }
	357	+
	358	+ $html = wp_remote_retrieve_body($response);
	359	+ $content = $this->mxchat_extract_main_content($html);
	360	+ $sanitized = $this->mxchat_sanitize_content_for_api($content);
	361	+
	362	+ if (empty($sanitized)) {
	363	+ return false;
	364	+ }
	365	+
	366	+ $options = get_option('mxchat_options');
	367	+ $api_key = $options['api_key'] ?? '';
	368	+
	369	+ $result = MxChat_Utils::submit_content_to_db($sanitized, $url, $api_key);
	370	+
	371	+ return !is_wp_error($result);
	372	+
	373	+ } catch (Exception $e) {
	374	+ //error_log('Single URL processing error: ' . $e->getMessage());
	375	+ return false;
	376	+ }
	377	+}
	378	+
	379	+
97	380	// ========================================
98	381	// MAIN CONTENT SUBMISSION HANDLERS
99	382	// ========================================
100	383
		@@ -110,18 +393,13 @@
110	393	wp_die(esc_html__('Nonce verification failed.', 'mxchat'));
111	394	}
112	395
113	396	// Sanitize the inputs.
114		- // Use wp_kses_post to allow safe HTML and wp_unslash to remove WordPress added slashes
115		- $article_content = wp_kses_post(wp_unslash($_POST['article_content']));
	397	+ $article_content = sanitize_textarea_field($_POST['article_content']);
116	398	$article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
117	399
118		- // Get bot_id from form submission
119		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
120		-
121		- // Get bot-specific options and API key
122		- $bot_options = $this->get_bot_options($bot_id);
123		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	400	+ // Get API key for submission
	401	+ $options = get_option('mxchat_options');
124	402	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
125	403
126	404	if (strpos($selected_model, 'voyage') === 0) {
127	405	$api_key = $options['voyage_api_key'] ?? '';
		@@ -139,10 +417,10 @@
139	417	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
140	418	exit;
141	419	}
142	420
143		- // Use centralized utility function with bot_id
144		- $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key, null, $bot_id);
	421	+ // Use centralized utility function for storage
	422	+ $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key);
145	423
146	424	if (is_wp_error($result)) {
147	425	set_transient('mxchat_admin_notice_error',
148	426	esc_html__('Error storing content: ', 'mxchat') . $result->get_error_message(),
		@@ -157,268 +435,17 @@
157	435
158	436	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
159	437	exit;
160	438	}
161		-
162		-/**
163		- * Handle the "YouTube" KB import source (admin-post form submission).
164		- *
165		- * Per-video description mode:
166		- * - auto: fetch oEmbed metadata (reliable) + best-effort captions transcript.
167		- * If no usable transcript, index the metadata anyway, tell the admin,
168		- * and bounce back with the manual box pre-filled (never fail silently).
169		- * - manual: the admin's own description is what gets indexed; metadata rides along.
170		- *
171		- * The row is stored with content_type 'youtube' and source_url = the canonical
172		- * watch URL, so re-importing the same video UPDATES the entry (source_url
173		- * duplicate handling in MxChat_Utils::store_in_wordpress_db) — that is also the
174		- * "augment a metadata-only entry" path.
175		- */
176		-public function mxchat_handle_youtube_submission() {
177		- if (!isset($_POST['submit_youtube']) || !current_user_can('manage_options')) {
178		- wp_die(esc_html__('Unauthorized access', 'mxchat'));
179		- }
180		-
181		- check_admin_referer('mxchat_submit_youtube_action', 'mxchat_submit_youtube_nonce');
182		-
183		- $redirect_url = admin_url('admin.php?page=mxchat-prompts');
184		-
185		- $youtube_url = isset($_POST['youtube_url']) ? esc_url_raw(wp_unslash($_POST['youtube_url'])) : '';
186		- $video_id = MxChat_Utils::parse_youtube_id($youtube_url);
187		-
188		- if (empty($video_id)) {
189		- set_transient('mxchat_admin_notice_error',
190		- esc_html__('That does not look like a link to a single YouTube video. Please paste a watch, youtu.be, or Shorts URL.', 'mxchat'),
191		- 30
192		- );
193		- wp_safe_redirect(esc_url($redirect_url));
194		- exit;
195		- }
196		-
197		- $canonical_url = 'https://www.youtube.com/watch?v=' . $video_id;
198		-
199		- $description_mode = (isset($_POST['youtube_description_mode']) && $_POST['youtube_description_mode'] === 'manual') ? 'manual' : 'auto';
200		- $manual_description = isset($_POST['youtube_description']) ? trim(wp_kses_post(wp_unslash($_POST['youtube_description']))) : '';
201		-
202		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
203		-
204		- // Resolve the embedding API key exactly like the sibling handlers.
205		- $bot_options = $this->get_bot_options($bot_id);
206		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
207		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
208		-
209		- if (strpos($selected_model, 'voyage') === 0) {
210		- $api_key = $options['voyage_api_key'] ?? '';
211		- $provider_name = 'Voyage AI';
212		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
213		- $api_key = $options['gemini_api_key'] ?? '';
214		- $provider_name = 'Google Gemini';
215		- } else {
216		- $api_key = $options['api_key'] ?? '';
217		- $provider_name = 'OpenAI';
218		- }
219		-
220		- if (empty($api_key)) {
221		- set_transient('mxchat_admin_notice_error',
222		- sprintf(
223		- esc_html__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
224		- $provider_name
225		- ),
226		- 30
227		- );
228		- wp_safe_redirect(esc_url($redirect_url));
229		- exit;
230		- }
231		-
232		- // Metadata is fetched in BOTH modes — it is the reliable half of auto, and in
233		- // manual mode it enriches the indexed text with the real title/channel.
234		- $meta = $this->mxchat_fetch_youtube_oembed($video_id);
235		- $video_title = isset($meta['title']) ? sanitize_text_field($meta['title']) : '';
236		- $video_channel = isset($meta['author_name']) ? sanitize_text_field($meta['author_name']) : '';
237		-
238		- $header_lines = 'YouTube Video: ' . ($video_title !== '' ? $video_title : $canonical_url) . "\n";
239		- if ($video_channel !== '') {
240		- $header_lines .= 'Channel: ' . $video_channel . "\n";
241		- }
242		- $header_lines .= 'URL: ' . $canonical_url . "\n\n";
243		-
244		- $transcript_missing = false;
245		-
246		- if ($description_mode === 'manual') {
247		- if ($manual_description === '') {
248		- set_transient('mxchat_admin_notice_error',
249		- esc_html__('Please write a description for the video, or switch to auto-fetch.', 'mxchat'),
250		- 30
251		- );
252		- wp_safe_redirect(esc_url($redirect_url));
253		- exit;
254		- }
255		- $indexed_text = $header_lines . $manual_description;
256		- } else {
257		- $transcript = $this->mxchat_fetch_youtube_transcript($video_id);
258		-
259		- if (strlen($transcript) >= 200) {
260		- $indexed_text = $header_lines . $transcript;
261		- } else {
262		- // Graceful fallback: captions disabled / blocked / no speech. Auto
263		- // reliably gets metadata; it does NOT guarantee a transcript.
264		- $transcript_missing = true;
265		-
266		- if ($video_title === '' && $video_channel === '') {
267		- // Both halves failed — nothing meaningful to index.
268		- set_transient('mxchat_admin_notice_error',
269		- esc_html__('Could not retrieve any information for that video (no metadata and no captions). Please check the URL, or use the manual description option.', 'mxchat'),
270		- 30
271		- );
272		- wp_safe_redirect(esc_url($redirect_url));
273		- exit;
274		- }
275		-
276		- $indexed_text = $header_lines . sprintf(
277		- /* translators: 1: video title, 2: channel name */
278		- __('A YouTube video titled "%1$s" from the channel %2$s.', 'mxchat'),
279		- $video_title !== '' ? $video_title : $canonical_url,
280		- $video_channel !== '' ? $video_channel : 'YouTube'
281		- );
282		- }
283		- }
284		-
285		- $result = MxChat_Utils::submit_content_to_db($indexed_text, $canonical_url, $api_key, null, $bot_id, 'youtube');
286		-
287		- if (is_wp_error($result)) {
288		- set_transient('mxchat_admin_notice_error',
289		- esc_html__('Error storing video in the knowledge base: ', 'mxchat') . $result->get_error_message(),
290		- 30
291		- );
292		- wp_safe_redirect(esc_url($redirect_url));
293		- exit;
294		- }
295		-
296		- if ($transcript_missing) {
297		- set_transient('mxchat_admin_notice_success',
298		- esc_html__('Video indexed from its title and channel — no captions were available for a transcript. The form below is pre-filled: write your own description and import again to improve matching (it updates the same entry).', 'mxchat'),
299		- 30
300		- );
301		- // Bounce back with prefill args so the page reopens the YouTube form in
302		- // manual mode with the URL + fetched title ready to augment.
303		- $redirect_url = add_query_arg(array(
304		- 'mxchat_yt_prefill' => '1',
305		- 'yt_url' => rawurlencode($canonical_url),
306		- 'yt_title' => rawurlencode($video_title),
307		- ), $redirect_url);
308		- } else {
309		- set_transient('mxchat_admin_notice_success',
310		- esc_html__('YouTube video successfully added to the knowledge base!', 'mxchat'),
311		- 30
312		- );
313		- }
314		-
315		- wp_safe_redirect(esc_url_raw($redirect_url));
316		- exit;
317		-}
318		-
319		-/**
320		- * Fetch YouTube oEmbed metadata for a video (no API key required).
321		- * Returns the decoded array (title, author_name, thumbnail_url, ...) or array().
322		- */
323		-private function mxchat_fetch_youtube_oembed($video_id) {
324		- $oembed_url = 'https://www.youtube.com/oembed?url=' . rawurlencode('https://www.youtube.com/watch?v=' . $video_id) . '&format=json';
325		- $response = wp_remote_get($oembed_url, array('timeout' => 15));
326		- if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
327		- return array();
328		- }
329		- $data = json_decode(wp_remote_retrieve_body($response), true);
330		- return is_array($data) ? $data : array();
331		-}
332		-
333		-/**
334		- * Best-effort captions transcript for a video. Deliberately ISOLATED: this uses
335		- * YouTube's unofficial timedtext route (the caption track list embedded in the
336		- * watch page), which YouTube has broken before and will break again. Every
337		- * failure mode returns '' so a break degrades to the metadata-only import path
338		- * instead of erroring the whole submission. Do not let anything in here throw.
339		- */
340		-private function mxchat_fetch_youtube_transcript($video_id) {
341		- $watch_url = 'https://www.youtube.com/watch?v=' . $video_id . '&hl=en';
342		-
343		- // First try the honest ingest UA; some responses omit the player config for
344		- // bot UAs, so retry once with a browser UA before giving up.
345		- $user_agents = array(
346		- mxchat_ingest_user_agent(),
347		- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
348		- );
349		-
350		- $tracks = array();
351		- foreach ($user_agents as $ua) {
352		- $response = wp_remote_get($watch_url, array(
353		- 'timeout' => 20,
354		- 'user-agent' => $ua,
355		- ));
356		- if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
357		- continue;
358		- }
359		- $body = wp_remote_retrieve_body($response);
360		- if (!is_string($body) || $body === '' || !preg_match('/"captionTracks":(\[.*?\])(?=,")/s', $body, $m)) {
361		- continue;
362		- }
363		- $decoded = json_decode($m[1], true);
364		- if (is_array($decoded) && !empty($decoded)) {
365		- $tracks = $decoded;
366		- break;
367		- }
368		- }
369		-
370		- if (empty($tracks)) {
371		- return '';
372		- }
373		-
374		- // Prefer an English track, else take the first offered.
375		- $chosen = null;
376		- foreach ($tracks as $track) {
377		- if (isset($track['languageCode']) && strpos($track['languageCode'], 'en') === 0) {
378		- $chosen = $track;
379		- break;
380		- }
381		- }
382		- if ($chosen === null) {
383		- $chosen = $tracks[0];
384		- }
385		- if (empty($chosen['baseUrl']) || !is_string($chosen['baseUrl'])) {
386		- return '';
387		- }
388		-
389		- $timedtext = wp_remote_get($chosen['baseUrl'], array('timeout' => 20));
390		- if (is_wp_error($timedtext) || wp_remote_retrieve_response_code($timedtext) !== 200) {
391		- return '';
392		- }
393		- $xml = wp_remote_retrieve_body($timedtext);
394		- if (!is_string($xml) || strpos($xml, '<text') === false) {
395		- return '';
396		- }
397		-
398		- // <text start=".." dur="..">caption</text> — strip tags, decode the
399		- // double-encoded entities timedtext ships, collapse whitespace.
400		- $text = preg_replace('/<[^>]+>/', ' ', $xml);
401		- $text = html_entity_decode(html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8'), ENT_QUOTES | ENT_HTML5, 'UTF-8');
402		- $text = trim(preg_replace('/\s+/u', ' ', $text));
403		-
404		- return $text;
405		-}
406		-
407	439	public function mxchat_is_pdf_url($url, $response) {
408	440	$content_type = wp_remote_retrieve_header($response, 'content-type');
409	441	$file_extension = strtolower(pathinfo($url, PATHINFO_EXTENSION));
410	442
411		- // Check Content-Disposition header for .pdf filename (Google Drive sends this)
412		- $disposition = wp_remote_retrieve_header($response, 'content-disposition');
413		- $has_pdf_disposition = ! empty($disposition) && stripos($disposition, '.pdf') !== false;
414		-
415		- return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf' || $has_pdf_disposition;
	443	+ return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf';
416	444	}
417		-
418		-
419		-public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response, $bot_id = 'default') {
	445	+public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response) {
420	446	if (!current_user_can('manage_options')) {
	447	+ //error_log(esc_html__('Unauthorized PDF processing attempt', 'mxchat'));
421	448	return false;
422	449	}
423	450
424	451	$pdf_url = esc_url_raw($pdf_url);
		@@ -424,8 +451,9 @@
424	451	$pdf_url = esc_url_raw($pdf_url);
425	452	$upload_dir = wp_upload_dir();
426	453
427	454	if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
	455	+ //error_log(sprintf(esc_html__('Upload directory error: %s', 'mxchat'), esc_html($upload_dir['error'])));
428	456	return false;
429	457	}
430	458
431	459	$pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
		@@ -432,12 +460,14 @@
432	460	$pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
433	461
434	462	$response_body = wp_remote_retrieve_body($response);
435	463	if (empty($response_body)) {
	464	+ //error_log(esc_html__('Empty PDF response body', 'mxchat'));
436	465	return false;
437	466	}
438	467
439	468	if (!wp_mkdir_p(dirname($pdf_path))) {
	469	+ //error_log(sprintf(esc_html__('Failed to create directory for PDF: %s', 'mxchat'), esc_html($pdf_path)));
440	470	return false;
441	471	}
442	472
443	473	try {
		@@ -446,915 +476,303 @@
446	476	if (!file_exists($pdf_path)) {
447	477	throw new Exception(__('Failed to save PDF file', 'mxchat'));
448	478	}
449	479
450		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
451		-
452		- if ($total_pages === false || $total_pages < 1) {
453		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
454		- }
	480	+ $parser = new \Smalot\PdfParser\Parser();
	481	+ $pdf = $parser->parseFile($pdf_path);
	482	+ $total_pages = absint(count($pdf->getPages()));
455	483
456		- // Create unique queue ID
457		- $queue_id = 'pdf_' . md5($pdf_url . time());
458		-
459		- // Create array of pages to process
460		- $pages = array();
461		- for ($i = 1; $i <= $total_pages; $i++) {
462		- $pages[] = array(
463		- 'pdf_path' => $pdf_path,
464		- 'pdf_url' => $pdf_url,
465		- 'page_number' => $i,
466		- 'total_pages' => $total_pages
467		- );
	484	+ if ($total_pages < 1) {
	485	+ throw new Exception(__('Invalid PDF: no pages found', 'mxchat'));
468	486	}
469	487
470		- // Add pages to queue
471		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
	488	+ wp_schedule_single_event(time(), 'mxchat_process_pdf_pages', array(
	489	+ 'pdf_path' => $pdf_path,
	490	+ 'pdf_url' => $pdf_url,
	491	+ 'total_pages' => $total_pages,
	492	+ 'batch_size' => absint(15),
	493	+ 'batch_pause' => absint(10)
	494	+ ));
472	495
473		- if ($queued_count === 0) {
474		- wp_delete_file($pdf_path);
475		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
476		- }
	496	+ $status_data = array(
	497	+ 'total_pages' => $total_pages,
	498	+ 'processed_pages' => 0,
	499	+ 'status' => 'processing',
	500	+ 'last_update' => time()
	501	+ );
477	502
478		- // Store queue metadata
479		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $pdf_url);
480		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
481		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
482		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
483		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
484		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	503	+ set_transient(
	504	+ sanitize_key('mxchat_pdf_status_' . md5($pdf_url)),
	505	+ array_map('sanitize_text_field', $status_data),
	506	+ DAY_IN_SECONDS
	507	+ );
485	508
486		- // Store queue ID in transient for status tracking
487		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
488		- set_transient('mxchat_last_pdf_url', $pdf_url, DAY_IN_SECONDS);
	509	+ return __('scheduled', 'mxchat');
489	510
490		- return 'queued';
491		-
492	511	} catch (Exception $e) {
	512	+ //error_log(sprintf(esc_html__('Error preparing PDF for processing: %s', 'mxchat'), esc_html($e->getMessage())));
493	513	if (file_exists($pdf_path)) {
494	514	wp_delete_file($pdf_path);
495	515	}
496		- return $e->getMessage();
	516	+ return false;
497	517	}
498	518	}
499	519
500		-/**
501		- * Handle direct PDF file upload from the knowledge base page
502		- */
503		-public function mxchat_handle_pdf_file_submission() {
504		- if (!isset($_POST['submit_pdf_file']) || !current_user_can('manage_options')) {
505		- wp_die(esc_html__('Unauthorized access', 'mxchat'));
506		- }
	520	+public function mxchat_process_pdf_pages_cron($pdf_path, $pdf_url, $total_pages, $batch_size, $batch_pause) {
	521	+ // Validate inputs
	522	+ $pdf_path = sanitize_text_field($pdf_path);
	523	+ $pdf_url = esc_url_raw($pdf_url);
	524	+ $total_pages = absint($total_pages);
	525	+ $batch_size = absint($batch_size);
	526	+ $batch_pause = absint($batch_pause);
507	527
508		- check_admin_referer('mxchat_submit_pdf_file_action', 'mxchat_submit_pdf_file_nonce');
509		-
510		- $redirect_url = admin_url('admin.php?page=mxchat-prompts');
511		-
512		- // Validate file upload
513		- if (empty($_FILES['pdf_file']) || $_FILES['pdf_file']['error'] !== UPLOAD_ERR_OK) {
514		- $error_code = isset($_FILES['pdf_file']['error']) ? $_FILES['pdf_file']['error'] : UPLOAD_ERR_NO_FILE;
515		- $error_messages = array(
516		- UPLOAD_ERR_INI_SIZE => __('The uploaded file exceeds the server upload_max_filesize limit.', 'mxchat'),
517		- UPLOAD_ERR_FORM_SIZE => __('The uploaded file exceeds the form MAX_FILE_SIZE limit.', 'mxchat'),
518		- UPLOAD_ERR_PARTIAL => __('The file was only partially uploaded.', 'mxchat'),
519		- UPLOAD_ERR_NO_FILE => __('No file was uploaded. Please select a PDF file.', 'mxchat'),
520		- UPLOAD_ERR_NO_TMP_DIR => __('Server missing temporary folder.', 'mxchat'),
521		- UPLOAD_ERR_CANT_WRITE => __('Server failed to write file to disk.', 'mxchat'),
522		- );
523		- $error_msg = isset($error_messages[$error_code]) ? $error_messages[$error_code] : __('Unknown upload error.', 'mxchat');
524		- set_transient('mxchat_admin_notice_error', $error_msg, 30);
525		- wp_safe_redirect(esc_url($redirect_url));
526		- exit;
527		- }
528		-
529		- $file = $_FILES['pdf_file'];
530		-
531		- // Validate MIME type
532		- $finfo = finfo_open(FILEINFO_MIME_TYPE);
533		- $mime_type = finfo_file($finfo, $file['tmp_name']);
534		- finfo_close($finfo);
535		-
536		- if ($mime_type !== 'application/pdf') {
537		- set_transient('mxchat_admin_notice_error',
538		- esc_html__('Invalid file type. Only PDF files are accepted.', 'mxchat'),
539		- 30
540		- );
541		- wp_safe_redirect(esc_url($redirect_url));
542		- exit;
543		- }
544		-
545		- // Validate extension
546		- $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
547		- if ($ext !== 'pdf') {
548		- set_transient('mxchat_admin_notice_error',
549		- esc_html__('Invalid file extension. Only .pdf files are accepted.', 'mxchat'),
550		- 30
551		- );
552		- wp_safe_redirect(esc_url($redirect_url));
553		- exit;
554		- }
555		-
556		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
557		- $original_filename = sanitize_file_name($file['name']);
558		-
559		- $upload_dir = wp_upload_dir();
560		- if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
561		- set_transient('mxchat_admin_notice_error',
562		- esc_html__('WordPress upload directory is not writable.', 'mxchat'),
563		- 30
564		- );
565		- wp_safe_redirect(esc_url($redirect_url));
566		- exit;
567		- }
568		-
569		- $pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
570		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
571		-
572		- if (!wp_mkdir_p(dirname($pdf_path))) {
573		- set_transient('mxchat_admin_notice_error',
574		- esc_html__('Failed to create upload directory.', 'mxchat'),
575		- 30
576		- );
577		- wp_safe_redirect(esc_url($redirect_url));
578		- exit;
579		- }
580		-
581		- // Move uploaded file
582		- if (!move_uploaded_file($file['tmp_name'], $pdf_path)) {
583		- set_transient('mxchat_admin_notice_error',
584		- esc_html__('Failed to save uploaded PDF file.', 'mxchat'),
585		- 30
586		- );
587		- wp_safe_redirect(esc_url($redirect_url));
588		- exit;
589		- }
590		-
591	528	try {
592		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
593		-
594		- if ($total_pages === false || $total_pages < 1) {
595		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
	529	+ if (!file_exists($pdf_path)) {
	530	+ throw new Exception(sprintf('PDF file not found at path: %s', esc_html($pdf_path)));
596	531	}
597	532
598		- // Use original filename as the source identifier
599		- $source_label = 'upload://' . $original_filename;
	533	+ $parser = new \Smalot\PdfParser\Parser();
	534	+ $pdf = $parser->parseFile($pdf_path);
	535	+ $pages = $pdf->getPages();
600	536
601		- $queue_id = 'pdf_' . md5($source_label . time());
	537	+ // Get current progress
	538	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	539	+ $status = get_transient($status_key);
602	540
603		- $pages = array();
604		- for ($i = 1; $i <= $total_pages; $i++) {
605		- $pages[] = array(
606		- 'pdf_path' => $pdf_path,
607		- 'pdf_url' => $source_label,
608		- 'page_number' => $i,
609		- 'total_pages' => $total_pages,
610		- );
	541	+ if (!$status || !is_array($status)) {
	542	+ throw new Exception('Invalid status data retrieved from transient');
611	543	}
612	544
613		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
	545	+ // Initialize failed pages list if it doesn't exist
	546	+ if (!isset($status['failed_pages_list']) || !is_array($status['failed_pages_list'])) {
	547	+ $status['failed_pages_list'] = [];
	548	+ }
614	549
615		- if ($queued_count === 0) {
616		- wp_delete_file($pdf_path);
617		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
	550	+ $start_page = absint($status['processed_pages']);
	551	+ $end_page = min($start_page + $batch_size, $total_pages);
	552	+ $options = get_option('mxchat_options');
	553	+
	554	+ if (empty($options['api_key'])) {
	555	+ throw new Exception('API key is missing or invalid');
618	556	}
619	557
620		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $source_label);
621		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
622		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
623		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
624		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
625		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	558	+ $successful_pages = 0;
	559	+ $failed_pages = 0;
626	560
627		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
628		- set_transient('mxchat_last_pdf_url', $source_label, DAY_IN_SECONDS);
	561	+ for ($i = $start_page; $i < $end_page; $i++) {
	562	+ $page_number = $i + 1;
	563	+ $max_retries = 3;
	564	+ $retry_count = 0;
	565	+ $page_processed = false;
	566	+ $last_error = '';
629	567
630		- set_transient('mxchat_admin_notice_success',
631		- sprintf(
632		- esc_html__('PDF "%s" (%d pages) queued for processing. Processing will start automatically.', 'mxchat'),
633		- esc_html($original_filename),
634		- $total_pages
635		- ),
636		- 30
637		- );
	568	+ while (!$page_processed && $retry_count < $max_retries) {
	569	+ try {
	570	+ $text = $pages[$i]->getText();
	571	+
	572	+ if (empty($text)) {
	573	+ throw new Exception("Empty text on page {$page_number}");
	574	+ }
	575	+
	576	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
638	577
639		- } catch (Exception $e) {
640		- if (file_exists($pdf_path)) {
641		- wp_delete_file($pdf_path);
642		- }
643		- set_transient('mxchat_admin_notice_error',
644		- esc_html__('Failed to process uploaded PDF: ', 'mxchat') . esc_html($e->getMessage()),
645		- 30
646		- );
647		- }
	578	+ if (empty($sanitized_content)) {
	579	+ throw new Exception("No valid content after sanitization on page {$page_number}");
	580	+ }
648	581
649		- wp_safe_redirect(esc_url($redirect_url));
650		- exit;
651		-}
	582	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	583	+
	584	+ if (is_string($embedding_vector)) {
	585	+ throw new Exception("Embedding generation failed: " . $embedding_vector);
	586	+ }
	587	+
	588	+ if (!is_array($embedding_vector)) {
	589	+ throw new Exception("Embedding generation returned unexpected result type: " . gettype($embedding_vector));
	590	+ }
	591	+
	592	+ $metadata = array(
	593	+ 'document_type' => 'pdf',
	594	+ 'total_pages' => $total_pages,
	595	+ 'current_page' => $page_number,
	596	+ 'prev_page' => $i > 0 ? $i : null,
	597	+ 'next_page' => $i < ($total_pages - 1) ? $i + 2 : null,
	598	+ 'source_url' => $pdf_url
	599	+ );
652	600
653		-/**
654		- * Validate PDF and count pages with multiple parser attempts
655		- */
656		-private function mxchat_validate_and_count_pdf_pages($pdf_path) {
657		- // Method 1: Try with Smalot PDF Parser (your current method)
658		- try {
659		- mxchat_load_pdf_parser();
660		- $parser = new \Smalot\PdfParser\Parser();
661		- $pdf = $parser->parseFile($pdf_path);
662		- $pages = $pdf->getPages();
663		- $page_count = count($pages);
664		-
665		- if ($page_count > 0) {
666		- //error_log('PDF parsed successfully with Smalot parser: ' . $page_count . ' pages');
667		- return $page_count;
668		- }
669		- } catch (Exception $e) {
670		- //error_log('Smalot PDF parser failed: ' . $e->getMessage());
671		- }
	601	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	602	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
672	603
673		- // Method 2: Try with pdfinfo command (if available)
674		- if (function_exists('shell_exec') && !$this->mxchat_is_shell_disabled()) {
675		- try {
676		- $command = 'pdfinfo ' . escapeshellarg($pdf_path) . ' 2>&1';
677		- $output = shell_exec($command);
678		-
679		- if ($output && preg_match('/Pages:\s*(\d+)/', $output, $matches)) {
680		- $page_count = intval($matches[1]);
681		- if ($page_count > 0) {
682		- //error_log('PDF parsed successfully with pdfinfo: ' . $page_count . ' pages');
683		- return $page_count;
	604	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $options['api_key']);
	605	+
	606	+ if (is_wp_error($db_result)) {
	607	+ throw new Exception("Database submission failed: " . $db_result->get_error_message());
	608	+ }
	609	+
	610	+ // Success!
	611	+ $page_processed = true;
	612	+ $successful_pages++;
	613	+
	614	+ } catch (Exception $e) {
	615	+ $retry_count++;
	616	+ $last_error = $e->getMessage();
	617	+
	618	+ //error_log("PDF page {$page_number} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	619	+
	620	+ if ($retry_count < $max_retries) {
	621	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	622	+ sleep(pow(2, $retry_count - 1));
	623	+ }
684	624	}
685	625	}
686		- } catch (Exception $e) {
687		- //error_log('pdfinfo command failed: ' . $e->getMessage());
688		- }
689		- }
690	626
691		- // Method 3: Try to repair PDF and parse again
692		- try {
693		- $repaired_path = $this->mxchat_attempt_pdf_repair($pdf_path);
694		- if ($repaired_path && $repaired_path !== $pdf_path) {
695		- mxchat_load_pdf_parser();
696		- $parser = new \Smalot\PdfParser\Parser();
697		- $pdf = $parser->parseFile($repaired_path);
698		- $pages = $pdf->getPages();
699		- $page_count = count($pages);
700		-
701		- if ($page_count > 0) {
702		- // Replace original with repaired version
703		- copy($repaired_path, $pdf_path);
704		- unlink($repaired_path);
705		- //error_log('PDF repaired and parsed successfully: ' . $page_count . ' pages');
706		- return $page_count;
	627	+ // If page still not processed after all retries, mark as failed
	628	+ if (!$page_processed) {
	629	+ $failed_pages++;
	630	+ $status['failed_pages_list'][] = [
	631	+ 'page' => $page_number,
	632	+ 'error' => $last_error,
	633	+ 'time' => time(),
	634	+ 'retries' => $max_retries
	635	+ ];
	636	+
	637	+ // Limit failed pages list to prevent memory issues
	638	+ if (count($status['failed_pages_list']) > 50) {
	639	+ $status['failed_pages_list'] = array_slice($status['failed_pages_list'], -50);
	640	+ }
	641	+
	642	+ //error_log("PDF page {$page_number} permanently failed after {$max_retries} attempts: " . $last_error);
707	643	}
	644	+
	645	+ // Update progress
	646	+ $status['processed_pages'] = absint($page_number);
	647	+ $status['last_update'] = time();
	648	+ $status['failed_pages'] = absint($status['failed_pages'] ?? 0) + ($page_processed ? 0 : 1);
708	649
709		- // Clean up repaired file if it didn't work
710		- unlink($repaired_path);
	650	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
711	651	}
712		- } catch (Exception $e) {
713		- //error_log('PDF repair attempt failed: ' . $e->getMessage());
714		- }
715	652
716		- // Method 4: Manual PDF structure analysis (basic page count)
717		- try {
718		- $page_count = $this->mxchat_manual_pdf_page_count($pdf_path);
719		- if ($page_count > 0) {
720		- //error_log('PDF page count determined manually: ' . $page_count . ' pages');
721		- return $page_count;
722		- }
723		- } catch (Exception $e) {
724		- //error_log('Manual PDF analysis failed: ' . $e->getMessage());
725		- }
726		-
727		- //error_log('All PDF parsing methods failed for: ' . $pdf_path);
728		- return false;
729		-}
730		-
731		-/**
732		- * Check if shell_exec is disabled
733		- */
734		-private function mxchat_is_shell_disabled() {
735		- $disabled = explode(',', ini_get('disable_functions'));
736		- return in_array('shell_exec', $disabled);
737		-}
738		-
739		-/**
740		- * Attempt to repair PDF using basic methods
741		- */
742		-private function mxchat_attempt_pdf_repair($pdf_path) {
743		- try {
744		- $content = file_get_contents($pdf_path);
745		- if (!$content) {
746		- return false;
747		- }
748		-
749		- // Check if PDF starts with proper header
750		- if (substr($content, 0, 4) !== '%PDF') {
751		- // Try to find PDF header in the content
752		- $header_pos = strpos($content, '%PDF');
753		- if ($header_pos !== false && $header_pos < 1024) {
754		- // Remove junk before PDF header
755		- $content = substr($content, $header_pos);
756		- $repaired_path = $pdf_path . '.repaired';
757		- file_put_contents($repaired_path, $content);
758		- return $repaired_path;
	653	+ // Schedule next batch if needed
	654	+ if ($end_page < $total_pages) {
	655	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_pdf_pages', array(
	656	+ 'pdf_path' => $pdf_path,
	657	+ 'pdf_url' => $pdf_url,
	658	+ 'total_pages' => $total_pages,
	659	+ 'batch_size' => $batch_size,
	660	+ 'batch_pause' => $batch_pause
	661	+ ));
	662	+ } else {
	663	+ // Processing complete
	664	+ $status['status'] = 'complete';
	665	+ $status['processed_pages'] = $total_pages;
	666	+
	667	+ // Add completion summary
	668	+ $status['completion_summary'] = [
	669	+ 'total_pages' => $total_pages,
	670	+ 'successful_pages' => $total_pages - absint($status['failed_pages'] ?? 0),
	671	+ 'failed_pages' => absint($status['failed_pages'] ?? 0),
	672	+ 'completion_time' => current_time('mysql')
	673	+ ];
	674	+
	675	+ // Save the completed status (don't delete it - let user dismiss manually)
	676	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	677	+
	678	+ // Clean up the temporary PDF file
	679	+ if (file_exists($pdf_path)) {
	680	+ wp_delete_file($pdf_path);
759	681	}
	682	+
	683	+ // DON'T delete the status transients here - let user dismiss manually
760	684	}
761	685
762		- // Check for EOF marker
763		- $content = rtrim($content);
764		- if (!preg_match('/%%EOF\s*$/', $content)) {
765		- // Add EOF marker if missing
766		- $content .= "\n%%EOF";
767		- $repaired_path = $pdf_path . '.repaired';
768		- file_put_contents($repaired_path, $content);
769		- return $repaired_path;
	686	+ } catch (\Exception $e) {
	687	+ //error_log(sprintf('[MXCHAT-PDF] Error processing PDF: %s', $e->getMessage()));
	688	+
	689	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	690	+ $status = get_transient($status_key);
	691	+
	692	+ if (!$status || !is_array($status)) {
	693	+ $status = array(
	694	+ 'total_pages' => $total_pages,
	695	+ 'processed_pages' => 0,
	696	+ 'status' => 'error',
	697	+ 'error' => sanitize_text_field($e->getMessage()),
	698	+ 'last_update' => time()
	699	+ );
	700	+ } else {
	701	+ $status['status'] = 'error';
	702	+ $status['error'] = sanitize_text_field($e->getMessage());
	703	+ $status['last_update'] = time();
770	704	}
771		-
772		- } catch (Exception $e) {
773		- //error_log('PDF repair error: ' . $e->getMessage());
774		- }
775		-
776		- return false;
777		-}
778		-
779		-/**
780		- * Manual PDF page counting by analyzing PDF structure
781		- */
782		-private function mxchat_manual_pdf_page_count($pdf_path) {
783		- try {
784		- $content = file_get_contents($pdf_path);
785		- if (!$content) {
786		- return 0;
	705	+
	706	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	707	+
	708	+ if (file_exists($pdf_path)) {
	709	+ wp_delete_file($pdf_path);
787	710	}
788		-
789		- // Method 1: Count /Type /Page objects
790		- $page_count = preg_match_all('/\/Type\s*\/Page[^s]/', $content);
791		- if ($page_count > 0) {
792		- return $page_count;
793		- }
794		-
795		- // Method 2: Look for /Count in pages object
796		- if (preg_match('/\/Type\s*\/Pages.*?\/Count\s+(\d+)/', $content, $matches)) {
797		- return intval($matches[1]);
798		- }
799		-
800		- // Method 3: Count page references
801		- $page_count = preg_match_all('/\d+\s+0\s+obj\s*<<[^>]*\/Type\s*\/Page/', $content);
802		- if ($page_count > 0) {
803		- return $page_count;
804		- }
805		-
806		- } catch (Exception $e) {
807		- //error_log('Manual PDF analysis error: ' . $e->getMessage());
808	711	}
809		-
810		- return 0;
811	712	}
812	713
813		-
814	714	public function mxchat_save_inline_prompt() {
815	715	// DEBUG: Log what we're receiving
816		- //error_log('=== MXCHAT DEBUG ===');
817		- //error_log('POST data: ' . print_r($_POST, true));
818		- //error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
	716	+ error_log('=== MXCHAT DEBUG ===');
	717	+ error_log('POST data: ' . print_r($_POST, true));
	718	+ error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
819	719
820	720	// Check for nonce security
821	721	check_ajax_referer('mxchat_save_inline_nonce', '_ajax_nonce');
822	722
823	723	// If we get here, nonce passed
824		- //error_log('Nonce verification PASSED');
825		-
826		- // Verify permissions
827		- if (!current_user_can('manage_options')) {
828		- wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
829		- return;
830		- }
831		-
832		- global $wpdb;
833		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
834		-
835		- // Validate and sanitize input data
836		- $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
837		- $article_content = isset($_POST['article_content']) ? wp_kses_post(wp_unslash($_POST['article_content'])) : '';
838		- $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
839		-
840		- if ($prompt_id > 0 && !empty($article_content)) {
841		- // Re-generate the embedding vector for the updated content
842		- $embedding_vector = $this->mxchat_generate_embedding($article_content);
843		- if (is_array($embedding_vector)) {
844		- // Serialize the embedding vector before storing it
845		- $embedding_vector_serialized = serialize($embedding_vector);
846		- // Update the prompt in the database
847		- $updated = $wpdb->update(
848		- $table_name,
849		- array(
850		- 'article_content' => $article_content,
851		- 'embedding_vector' => $embedding_vector_serialized,
852		- 'source_url' => $article_url,
853		- ),
854		- array('id' => $prompt_id),
855		- array('%s', '%s', '%s'),
856		- array('%d')
857		- );
858		- if ($updated !== false) {
859		- wp_send_json_success();
860		- } else {
861		- MxChat_Admin::mxchat_log_debug('knowledge_error', 'Database update failed when saving knowledge entry');
862		- wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
863		- }
864		- } else {
865		- MxChat_Admin::mxchat_log_debug('embedding_error', 'Embedding generation failed for knowledge entry');
866		- wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
867		- }
868		- } else {
869		- wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
870		- }
871		-}
	724	+ error_log('Nonce verification PASSED');
872	725
	726	+ // Verify permissions
	727	+ if (!current_user_can('manage_options')) {
	728	+ wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
	729	+ return;
	730	+ }
873	731
874		-/**
875		- * AJAX: Get full content for editing — reassembles chunks if needed.
876		- * Works for both WordPress DB and Pinecone entries.
877		- */
878		-public function ajax_mxchat_get_entry_content() {
879		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
	732	+ global $wpdb;
	733	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
880	734
881		- if ( ! current_user_can('manage_options') ) {
882		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
883		- }
	735	+ // Validate and sanitize input data
	736	+ $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
	737	+ $article_content = isset($_POST['article_content']) ? sanitize_textarea_field($_POST['article_content']) : '';
	738	+ $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
884	739
885		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
886		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
887		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
888		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
	740	+ if ($prompt_id > 0 && !empty($article_content)) {
	741	+ // Re-generate the embedding vector for the updated content
	742	+ $embedding_vector = $this->mxchat_generate_embedding($article_content);
889	743
890		- if ( $data_source === 'pinecone' ) {
891		- // Pinecone: fetch vectors by source_url, reassemble chunks
892		- $content = $this->get_pinecone_entry_content( $source_url, $entry_id, $bot_id );
893		- } else {
894		- // WordPress DB
895		- $content = $this->get_wordpress_entry_content( $source_url, $entry_id );
896		- }
	744	+ if (is_array($embedding_vector)) {
	745	+ // Serialize the embedding vector before storing it
	746	+ $embedding_vector_serialized = serialize($embedding_vector);
897	747
898		- if ( is_wp_error( $content ) ) {
899		- wp_send_json_error( array( 'message' => $content->get_error_message() ) );
900		- }
	748	+ // Update the prompt in the database
	749	+ $updated = $wpdb->update(
	750	+ $table_name,
	751	+ array(
	752	+ 'article_content' => $article_content,
	753	+ 'embedding_vector' => $embedding_vector_serialized,
	754	+ 'source_url' => $article_url,
	755	+ ),
	756	+ array('id' => $prompt_id),
	757	+ array('%s', '%s', '%s'),
	758	+ array('%d')
	759	+ );
901	760
902		- wp_send_json_success( $content );
903		-}
	761	+ if ($updated !== false) {
	762	+ wp_send_json_success();
	763	+ } else {
	764	+ wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
	765	+ }
	766	+ } else {
	767	+ wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
	768	+ }
	769	+ } else {
	770	+ wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
	771	+ }
	772	+ }
904	773
905		-/**
906		- * Get content from WordPress DB — reassembles chunks by source_url.
907		- */
908		-private function get_wordpress_entry_content( $source_url, $entry_id ) {
909		- global $wpdb;
910		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
911	774
912		- // If we have a source_url, check for chunks
913		- if ( ! empty( $source_url ) && strpos( $source_url, 'mxchat://' ) !== 0 ) {
914		- $rows = $wpdb->get_results( $wpdb->prepare(
915		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
916		- $source_url
917		- ) );
918		-
919		- if ( $rows && count( $rows ) > 1 ) {
920		- // Multiple rows = chunked. Reassemble.
921		- $chunks = array();
922		- foreach ( $rows as $row ) {
923		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
924		- $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
925		- $chunks[ $index ] = $parsed['text'];
926		- }
927		- ksort( $chunks );
928		- return array(
929		- 'content' => implode( "\n\n", $chunks ),
930		- 'source_url' => $source_url,
931		- 'is_chunked' => true,
932		- 'chunk_count' => count( $chunks ),
933		- 'content_type' => $rows[0]->content_type,
934		- );
935		- } elseif ( $rows && count( $rows ) === 1 ) {
936		- $parsed = MxChat_Chunker::parse_stored_chunk( $rows[0]->article_content );
937		- return array(
938		- 'content' => $parsed['text'],
939		- 'source_url' => $source_url,
940		- 'entry_id' => $rows[0]->id,
941		- 'is_chunked' => false,
942		- 'content_type' => $rows[0]->content_type,
943		- );
944		- }
945		- }
946		-
947		- // Fallback: fetch by ID
948		- if ( $entry_id > 0 ) {
949		- $row = $wpdb->get_row( $wpdb->prepare(
950		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
951		- $entry_id
952		- ) );
953		- if ( $row ) {
954		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
955		- return array(
956		- 'content' => $parsed['text'],
957		- 'source_url' => $row->source_url,
958		- 'entry_id' => $row->id,
959		- 'is_chunked' => false,
960		- 'content_type' => $row->content_type,
961		- );
962		- }
963		- }
964		-
965		- return new WP_Error( 'not_found', 'Entry not found.' );
966		-}
967		-
968		-/**
969		- * Get content from Pinecone — fetches vectors by source_url, reassembles chunks.
970		- */
971		-private function get_pinecone_entry_content( $source_url, $entry_id, $bot_id ) {
972		- if ( ! class_exists('MxChat_Pinecone_Manager') ) {
973		- return new WP_Error( 'pinecone_unavailable', 'Pinecone manager not available.' );
974		- }
975		-
976		- // Get Pinecone config
977		- if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
978		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
979		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
980		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
981		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
982		- } else {
983		- $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
984		- $api_key = $bot_config['api_key'] ?? '';
985		- $host = $bot_config['host'] ?? '';
986		- $namespace = $bot_config['namespace'] ?? '';
987		- }
988		-
989		- if ( empty($host) || empty($api_key) ) {
990		- return new WP_Error( 'pinecone_config', 'Pinecone not configured.' );
991		- }
992		-
993		- // List vectors with the source_url prefix
994		- $base_id = md5( $source_url );
995		- $vector_ids = array( $base_id );
996		-
997		- // Find chunk vectors
998		- $list_url = "https://{$host}/vectors/list";
999		- $list_body = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
1000		- if ( ! empty($namespace) ) {
1001		- $list_body['namespace'] = $namespace;
1002		- }
1003		-
1004		- $list_resp = wp_remote_post( $list_url, array(
1005		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1006		- 'body' => wp_json_encode( $list_body ),
1007		- 'timeout' => 15,
1008		- ) );
1009		-
1010		- if ( ! is_wp_error($list_resp) ) {
1011		- $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
1012		- if ( ! empty($list_data['vectors']) ) {
1013		- foreach ( $list_data['vectors'] as $v ) {
1014		- $vector_ids[] = $v['id'];
1015		- }
1016		- }
1017		- }
1018		-
1019		- // Fetch vectors with metadata
1020		- $fetch_url = "https://{$host}/vectors/fetch";
1021		- $fetch_body = array( 'ids' => $vector_ids );
1022		- if ( ! empty($namespace) ) {
1023		- $fetch_body['namespace'] = $namespace;
1024		- }
1025		-
1026		- $fetch_resp = wp_remote_post( $fetch_url, array(
1027		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1028		- 'body' => wp_json_encode( $fetch_body ),
1029		- 'timeout' => 15,
1030		- ) );
1031		-
1032		- if ( is_wp_error($fetch_resp) ) {
1033		- return new WP_Error( 'pinecone_fetch', 'Failed to fetch from Pinecone.' );
1034		- }
1035		-
1036		- $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
1037		- $vectors = $fetch_data['vectors'] ?? array();
1038		-
1039		- if ( empty($vectors) ) {
1040		- return new WP_Error( 'not_found', 'Entry not found in Pinecone.' );
1041		- }
1042		-
1043		- // Reassemble chunks
1044		- $chunks = array();
1045		- $content_type = 'content';
1046		- foreach ( $vectors as $vid => $vector ) {
1047		- $meta = $vector['metadata'] ?? array();
1048		- $text = $meta['text'] ?? '';
1049		- $index = $meta['chunk_index'] ?? 0;
1050		- $content_type = $meta['type'] ?? 'content';
1051		- $chunks[ intval($index) ] = $text;
1052		- }
1053		- ksort( $chunks );
1054		-
1055		- return array(
1056		- 'content' => implode( "\n\n", $chunks ),
1057		- 'source_url' => $source_url,
1058		- 'is_chunked' => count($chunks) > 1,
1059		- 'chunk_count' => count($chunks),
1060		- 'content_type' => $content_type,
1061		- );
1062		-}
1063		-
1064		-/**
1065		- * AJAX: Inspect a knowledge entry — returns the per-chunk stored text + metadata
1066		- * WITHOUT collapsing it, so a site owner can see exactly what was indexed for an
1067		- * entry (plan-mxchat-20260628-d8cb4b). READ-ONLY: never re-embeds or mutates.
1068		- */
1069		-public function ajax_mxchat_inspect_entry() {
1070		- check_ajax_referer('mxchat_inspect_entry_nonce', 'nonce');
1071		-
1072		- if ( ! current_user_can('manage_options') ) {
1073		- wp_send_json_error( array( 'message' => esc_html__('Permission denied.', 'mxchat') ) );
1074		- }
1075		-
1076		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
1077		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
1078		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
1079		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1080		-
1081		- if ( $data_source === 'pinecone' ) {
1082		- $result = $this->inspect_pinecone_entry( $source_url, $entry_id, $bot_id );
1083		- } else {
1084		- $result = $this->inspect_wordpress_entry( $source_url, $entry_id );
1085		- }
1086		-
1087		- if ( is_wp_error( $result ) ) {
1088		- wp_send_json_error( array( 'message' => $result->get_error_message() ) );
1089		- }
1090		-
1091		- wp_send_json_success( $result );
1092		-}
1093		-
1094		-/**
1095		- * Read-only inspector for WordPress-DB entries. Mirrors get_wordpress_entry_content()
1096		- * but returns each STORED chunk's exact text + length (no implode), plus the assembled
1097		- * embedded text. This shows what is actually in the index, not a re-derivation from the post.
1098		- */
1099		-private function inspect_wordpress_entry( $source_url, $entry_id ) {
1100		- global $wpdb;
1101		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
1102		-
1103		- $rows = array();
1104		-
1105		- // Group by the real stored source_url — this INCLUDES "mxchat://" manual
1106		- // Direct Content entries (the spec's manual-entry case), which share one
1107		- // source_url across their chunk rows. Only the synthetic "_ungrouped_<id>"
1108		- // display key (invented by the table view for rows with no source_url) is
1109		- // excluded; those fall through to the entry_id lookup below.
1110		- if ( ! empty( $source_url ) && strpos( $source_url, '_ungrouped_' ) !== 0 ) {
1111		- $rows = $wpdb->get_results( $wpdb->prepare(
1112		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
1113		- $source_url
1114		- ) );
1115		- }
1116		-
1117		- // Fallback / manual "Direct Content" entries: fetch the single row by id.
1118		- if ( empty( $rows ) && $entry_id > 0 ) {
1119		- $row = $wpdb->get_row( $wpdb->prepare(
1120		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
1121		- $entry_id
1122		- ) );
1123		- if ( $row ) {
1124		- $rows = array( $row );
1125		- }
1126		- }
1127		-
1128		- if ( empty( $rows ) ) {
1129		- return new WP_Error( 'not_found', esc_html__('Entry not found in the local knowledge database.', 'mxchat') );
1130		- }
1131		-
1132		- $chunks = array();
1133		- $content_type = '';
1134		- foreach ( $rows as $row ) {
1135		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
1136		- $text = isset( $parsed['text'] ) ? $parsed['text'] : '';
1137		- $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
1138		- $content_type = $row->content_type;
1139		- $chunks[] = array(
1140		- 'index' => $index,
1141		- 'text' => $text,
1142		- 'length' => function_exists('mb_strlen') ? mb_strlen( $text ) : strlen( $text ),
1143		- 'row_id' => intval( $row->id ),
1144		- );
1145		- }
1146		-
1147		- usort( $chunks, function( $a, $b ) { return $a['index'] - $b['index']; } );
1148		-
1149		- $assembled = implode( "\n\n", wp_list_pluck( $chunks, 'text' ) );
1150		-
1151		- return array(
1152		- 'store' => 'wordpress',
1153		- 'source_url' => $source_url,
1154		- 'content_type' => $content_type,
1155		- 'is_chunked' => count( $chunks ) > 1,
1156		- 'chunk_count' => count( $chunks ),
1157		- 'assembled' => $assembled,
1158		- 'assembled_length' => function_exists('mb_strlen') ? mb_strlen( $assembled ) : strlen( $assembled ),
1159		- 'chunks' => array_values( $chunks ),
1160		- // WP-DB storage carries no separate vector metadata; surface that fact
1161		- // rather than letting the owner guess (the spec's taxonomy question).
1162		- 'metadata' => array(),
1163		- 'metadata_note' => esc_html__('Stored in the local WordPress database. Only the assembled text shown here is embedded — there are no separate vector metadata fields (e.g. taxonomy terms are not stored unless they were injected into the text itself).', 'mxchat'),
1164		- );
1165		-}
1166		-
1167		-/**
1168		- * Read-only inspector for Pinecone entries. Mirrors get_pinecone_entry_content()
1169		- * but keeps each vector's text + metadata instead of imploding, so the owner can
1170		- * confirm exactly which metadata fields (text/source_url/type/last_updated/created_at/bot_id)
1171		- * are present per chunk. READ-ONLY.
1172		- */
1173		-private function inspect_pinecone_entry( $source_url, $entry_id, $bot_id ) {
1174		- if ( ! class_exists('MxChat_Pinecone_Manager') ) {
1175		- return new WP_Error( 'pinecone_unavailable', esc_html__('Pinecone manager not available.', 'mxchat') );
1176		- }
1177		-
1178		- if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
1179		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
1180		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1181		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1182		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1183		- } else {
1184		- $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1185		- $api_key = $bot_config['api_key'] ?? '';
1186		- $host = $bot_config['host'] ?? '';
1187		- $namespace = $bot_config['namespace'] ?? '';
1188		- }
1189		-
1190		- if ( empty($host) || empty($api_key) ) {
1191		- return new WP_Error( 'pinecone_config', esc_html__('Pinecone not configured.', 'mxchat') );
1192		- }
1193		-
1194		- $base_id = md5( $source_url );
1195		- $vector_ids = array( $base_id );
1196		-
1197		- $list_url = "https://{$host}/vectors/list";
1198		- $list_body = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
1199		- if ( ! empty($namespace) ) {
1200		- $list_body['namespace'] = $namespace;
1201		- }
1202		-
1203		- $list_resp = wp_remote_post( $list_url, array(
1204		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1205		- 'body' => wp_json_encode( $list_body ),
1206		- 'timeout' => 15,
1207		- ) );
1208		-
1209		- if ( ! is_wp_error($list_resp) ) {
1210		- $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
1211		- if ( ! empty($list_data['vectors']) ) {
1212		- foreach ( $list_data['vectors'] as $v ) {
1213		- $vector_ids[] = $v['id'];
1214		- }
1215		- }
1216		- }
1217		-
1218		- $fetch_url = "https://{$host}/vectors/fetch";
1219		- $fetch_body = array( 'ids' => $vector_ids );
1220		- if ( ! empty($namespace) ) {
1221		- $fetch_body['namespace'] = $namespace;
1222		- }
1223		-
1224		- $fetch_resp = wp_remote_post( $fetch_url, array(
1225		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1226		- 'body' => wp_json_encode( $fetch_body ),
1227		- 'timeout' => 15,
1228		- ) );
1229		-
1230		- if ( is_wp_error($fetch_resp) ) {
1231		- return new WP_Error( 'pinecone_fetch', esc_html__('Failed to fetch from Pinecone.', 'mxchat') );
1232		- }
1233		-
1234		- $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
1235		- $vectors = $fetch_data['vectors'] ?? array();
1236		-
1237		- if ( empty($vectors) ) {
1238		- return new WP_Error( 'not_found', esc_html__('Entry not found in Pinecone.', 'mxchat') );
1239		- }
1240		-
1241		- // Whitelisted metadata fields the spec calls out — shown so devs can confirm
1242		- // what is (and is NOT) stored per vector.
1243		- $meta_fields = array( 'text', 'source_url', 'type', 'last_updated', 'created_at', 'bot_id', 'chunk_index', 'total_chunks' );
1244		- $chunks = array();
1245		- $content_type = '';
1246		- foreach ( $vectors as $vid => $vector ) {
1247		- $meta = isset($vector['metadata']) && is_array($vector['metadata']) ? $vector['metadata'] : array();
1248		- $text = $meta['text'] ?? '';
1249		- $index = isset($meta['chunk_index']) ? intval($meta['chunk_index']) : count($chunks);
1250		- $content_type = $meta['type'] ?? $content_type;
1251		-
1252		- $clean_meta = array();
1253		- foreach ( $meta_fields as $field ) {
1254		- if ( array_key_exists( $field, $meta ) && $field !== 'text' ) {
1255		- $clean_meta[ $field ] = is_scalar( $meta[ $field ] ) ? (string) $meta[ $field ] : wp_json_encode( $meta[ $field ] );
1256		- }
1257		- }
1258		-
1259		- $chunks[] = array(
1260		- 'index' => $index,
1261		- 'text' => $text,
1262		- 'length' => function_exists('mb_strlen') ? mb_strlen( $text ) : strlen( $text ),
1263		- 'vector_id' => (string) $vid,
1264		- 'metadata' => $clean_meta,
1265		- );
1266		- }
1267		-
1268		- usort( $chunks, function( $a, $b ) { return $a['index'] - $b['index']; } );
1269		-
1270		- $assembled = implode( "\n\n", wp_list_pluck( $chunks, 'text' ) );
1271		-
1272		- return array(
1273		- 'store' => 'pinecone',
1274		- 'source_url' => $source_url,
1275		- 'content_type' => $content_type,
1276		- 'is_chunked' => count( $chunks ) > 1,
1277		- 'chunk_count' => count( $chunks ),
1278		- 'assembled' => $assembled,
1279		- 'assembled_length' => function_exists('mb_strlen') ? mb_strlen( $assembled ) : strlen( $assembled ),
1280		- 'chunks' => array_values( $chunks ),
1281		- 'metadata' => array(),
1282		- 'metadata_note' => esc_html__('Stored in Pinecone. Each chunk above lists the vector metadata fields actually present — if a field you expect (such as taxonomy terms) is missing here, it was not stored as metadata and is only searchable if it appears in the embedded text.', 'mxchat'),
1283		- );
1284		-}
1285		-
1286		-/**
1287		- * AJAX: Save edited content — re-chunks and re-embeds as needed.
1288		- * Works for both WordPress DB and Pinecone entries.
1289		- */
1290		-public function ajax_mxchat_save_entry_content() {
1291		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
1292		-
1293		- if ( ! current_user_can('manage_options') ) {
1294		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
1295		- }
1296		-
1297		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
1298		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
1299		- $content = isset($_POST['content']) ? wp_kses_post( wp_unslash($_POST['content']) ) : '';
1300		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
1301		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1302		- $content_type = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : 'content';
1303		-
1304		- if ( empty($content) ) {
1305		- wp_send_json_error( array( 'message' => 'Content cannot be empty.' ) );
1306		- }
1307		-
1308		- // Get the embedding API key
1309		- $options = get_option('mxchat_options', array());
1310		- $api_key = '';
1311		-
1312		- if ( $bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager') ) {
1313		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
1314		- $api_key = $bot_options['api_key'] ?? '';
1315		- }
1316		- if ( empty($api_key) ) {
1317		- $api_key = $options['api_key'] ?? '';
1318		- }
1319		-
1320		- global $wpdb;
1321		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
1322		-
1323		- // If source_url is empty but we have an entry_id, look it up
1324		- if ( empty($source_url) && $entry_id > 0 && $data_source === 'wordpress' ) {
1325		- $row = $wpdb->get_row( $wpdb->prepare( "SELECT source_url FROM {$table} WHERE id = %d", $entry_id ) );
1326		- if ( $row && ! empty($row->source_url) ) {
1327		- $source_url = $row->source_url;
1328		- }
1329		- }
1330		-
1331		- // For manual entries (no source_url or mxchat:// prefix), delete the old entry by ID first
1332		- // so submit_content_to_db creates a replacement instead of a duplicate
1333		- // Also treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
1334		- $is_legacy_manual = !empty($source_url) && strpos($source_url, 'mxchat.ai') !== false && strpos($source_url, 'mxchat://') !== 0;
1335		- if ( $entry_id > 0 && (empty($source_url) || strpos($source_url, 'mxchat://') === 0 || $is_legacy_manual) ) {
1336		- $wpdb->delete( $table, array( 'id' => $entry_id ), array( '%d' ) );
1337		- // Clear legacy URL so submit_content_to_db generates a unique mxchat:// identifier
1338		- // instead of reusing the shared URL (which would mass-delete other entries with the same URL)
1339		- if ( $is_legacy_manual ) {
1340		- $source_url = '';
1341		- }
1342		- }
1343		-
1344		- // Use the existing submit_content_to_db which handles chunking, Pinecone, and WP DB
1345		- $vector_id = ! empty($source_url) ? md5($source_url) : md5('mxchat_manual_' . $entry_id);
1346		-
1347		- // submit_content_to_db already handles: delete old chunks → re-chunk → re-embed → store
1348		- $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, $vector_id, $bot_id, $content_type );
1349		-
1350		- if ( is_wp_error($result) ) {
1351		- wp_send_json_error( array( 'message' => $result->get_error_message() ) );
1352		- }
1353		-
1354		- wp_send_json_success( array( 'message' => 'Content saved and re-embedded successfully.' ) );
1355		-}
1356		-
1357	775	public function mxchat_get_pdf_processing_status($pdf_url) {
1358	776	$pdf_url = esc_url_raw($pdf_url);
1359	777	$status = get_transient(sanitize_key('mxchat_pdf_status_' . md5($pdf_url)));
1360	778
		@@ -1397,18 +815,24 @@
1397	815	}
1398	816
1399	817
1400	818	public function mxchat_handle_sitemap_submission() {
	819	+ // Start logging the submission process
	820	+ //error_log('[MXCHAT-URL] ===== Starting URL submission process =====');
	821	+
1401	822	// Check if the form was submitted and verify permissions
1402	823	if (!isset($_POST['submit_sitemap']) || !current_user_can('manage_options')) {
	824	+ //error_log('[MXCHAT-URL] Error: Unauthorized access or form not submitted properly');
1403	825	wp_die(esc_html__('Unauthorized access', 'mxchat'));
1404	826	}
1405	827
1406	828	// Verify nonce
	829	+ //error_log('[MXCHAT-URL] Verifying nonce');
1407	830	check_admin_referer('mxchat_submit_sitemap_action', 'mxchat_submit_sitemap_nonce');
1408	831
1409	832	// Validate URL
1410	833	if (!isset($_POST['sitemap_url']) || empty($_POST['sitemap_url'])) {
	834	+ //error_log('[MXCHAT-URL] Error: Empty or missing URL');
1411	835	set_transient('mxchat_admin_notice_error',
1412	836	esc_html__('Please provide a valid URL.', 'mxchat'),
1413	837	30
1414	838	);
		@@ -1416,28 +840,12 @@
1416	840	exit;
1417	841	}
1418	842
1419	843	$submitted_url = esc_url_raw($_POST['sitemap_url']);
	844	+ //error_log('[MXCHAT-URL] Processing URL: ' . $submitted_url);
1420	845
1421		- // Convert Google Drive sharing URLs to direct download URLs
1422		- if ( strpos($submitted_url, 'drive.google.com') !== false ) {
1423		- $file_id = '';
1424		- if ( preg_match('/[?&]id=([a-zA-Z0-9_-]+)/', $submitted_url, $m) ) {
1425		- $file_id = $m[1];
1426		- } elseif ( preg_match('#/file/d/([a-zA-Z0-9_-]+)#', $submitted_url, $m) ) {
1427		- $file_id = $m[1];
1428		- }
1429		- if ( ! empty($file_id) ) {
1430		- $submitted_url = 'https://drive.google.com/uc?export=download&id=' . $file_id;
1431		- }
1432		- }
1433		-
1434		- // Get bot_id from form submission
1435		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1436		-
1437		- // Get bot-specific options and validate API key
1438		- $bot_options = $this->get_bot_options($bot_id);
1439		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	846	+ // Validate API key first
	847	+ $options = get_option('mxchat_options');
1440	848	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
1441	849
1442	850	if (strpos($selected_model, 'voyage') === 0) {
1443	851	$api_key = $options['voyage_api_key'] ?? '';
		@@ -1454,31 +862,21 @@
1454	862	$error_message = sprintf(
1455	863	esc_html__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
1456	864	$provider_name
1457	865	);
	866	+ //error_log('[MXCHAT-URL] Error: ' . $error_message);
1458	867	set_transient('mxchat_admin_notice_error', $error_message, 30);
	868	+ //error_log('[MXCHAT-URL] Set error transient: ' . $error_message);
1459	869	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1460	870	exit;
1461	871	}
1462	872
1463		- // Fetch URL — send an honest, versioned MXChat crawler UA (not a spoofed
1464		- // browser). Stale browser UAs are exactly what WAFs like SiteGround's
1465		- // ModSecurity flag as scrapers, 403-ing the fetch (including PDFs served
1466		- // from the site's own media library, which route through this same call).
1467		- // See mxchat_ingest_user_agent(). Accept is kept for content negotiation;
1468		- // the browser-only Accept-Language fingerprint is dropped so it stays
1469		- // coherent with a bot identity.
1470		- $response = wp_remote_get($submitted_url, array(
1471		- 'timeout' => 30,
1472		- 'sslverify' => false,
1473		- 'user-agent' => mxchat_ingest_user_agent(),
1474		- 'headers' => array(
1475		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1476		- ),
1477		- ));
	873	+ //error_log('[MXCHAT-URL] Fetching URL content');
	874	+ $response = wp_remote_get($submitted_url, array('timeout' => 30));
1478	875
1479	876	if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
1480	877	$error_message = is_wp_error($response) ? $response->get_error_message() : 'HTTP Status: ' . wp_remote_retrieve_response_code($response);
	878	+ //error_log('[MXCHAT-URL] Error fetching URL: ' . $error_message);
1481	879	set_transient('mxchat_admin_notice_error',
1482	880	sprintf(
1483	881	esc_html__('Failed to fetch the URL: %s', 'mxchat'),
1484	882	esc_html($error_message)
		@@ -1489,11 +887,13 @@
1489	887	exit;
1490	888	}
1491	889
1492	890	$content_type = wp_remote_retrieve_header($response, 'content-type');
	891	+ //error_log('[MXCHAT-URL] Content type: ' . $content_type);
1493	892	$body_content = wp_remote_retrieve_body($response);
1494	893
1495	894	if (empty($body_content)) {
	895	+ //error_log('[MXCHAT-URL] Error: Empty response body');
1496	896	set_transient('mxchat_admin_notice_error',
1497	897	esc_html__('Empty response received from URL.', 'mxchat'),
1498	898	30
1499	899	);
		@@ -1499,21 +899,29 @@
1499	899	);
1500	900	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1501	901	exit;
1502	902	}
	903	+ //error_log('[MXCHAT-URL] Retrieved body content length: ' . strlen($body_content) . ' bytes');
1503	904
1504	905	// Handle PDF URL
1505	906	if ($this->mxchat_is_pdf_url($submitted_url, $response)) {
1506		- $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response, $bot_id);
	907	+ //error_log('[MXCHAT-URL] Detected PDF URL, handling PDF for knowledge base');
	908	+ $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response);
	909	+ //error_log('[MXCHAT-URL] PDF handling result: ' . $result);
1507	910
1508		- if ($result === 'queued') {
1509		- set_transient('mxchat_admin_notice_success',
1510		- esc_html__('PDF queued for processing. Processing will start automatically.', 'mxchat'),
	911	+ if ($result === 'scheduled') {
	912	+ set_transient(
	913	+ 'mxchat_last_pdf_url',
	914	+ sanitize_text_field($submitted_url),
	915	+ DAY_IN_SECONDS
	916	+ );
	917	+ set_transient('mxchat_admin_notice_info',
	918	+ esc_html__('PDF processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1511	919	30
1512	920	);
1513	921	} else {
1514	922	set_transient('mxchat_admin_notice_error',
1515		- esc_html__('Failed to queue PDF processing: ', 'mxchat') . esc_html($result),
	923	+ esc_html__('Failed to start PDF processing: ', 'mxchat') . esc_html($result),
1516	924	30
1517	925	);
1518	926	}
1519	927
		@@ -1522,8 +930,9 @@
1522	930	}
1523	931
1524	932	// Handle Sitemap XML
1525	933	if (strpos($content_type, 'xml') !== false || strpos($body_content, '<urlset') !== false) {
	934	+ //error_log('[MXCHAT-URL] Detected XML content, processing as sitemap');
1526	935	libxml_use_internal_errors(true);
1527	936	$xml = simplexml_load_string($body_content);
1528	937	$xml_errors = libxml_get_errors();
1529	938	libxml_clear_errors();
		@@ -1528,8 +937,15 @@
1528	937	$xml_errors = libxml_get_errors();
1529	938	libxml_clear_errors();
1530	939
1531	940	if ($xml === false || !empty($xml_errors)) {
	941	+ //error_log('[MXCHAT-URL] Error: Invalid XML format');
	942	+ if (!empty($xml_errors)) {
	943	+ foreach ($xml_errors as $error) {
	944	+ //error_log('[MXCHAT-URL] XML Error: ' . $error->message);
	945	+ }
	946	+ }
	947	+
1532	948	set_transient('mxchat_admin_notice_error',
1533	949	esc_html__('Invalid sitemap XML. Please provide a valid sitemap.', 'mxchat'),
1534	950	30
1535	951	);
		@@ -1536,18 +952,27 @@
1536	952	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1537	953	exit;
1538	954	}
1539	955
1540		- $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url, $bot_id);
	956	+ //error_log('[MXCHAT-URL] Valid XML found, handling sitemap for knowledge base');
	957	+ $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url);
	958	+ //error_log('[MXCHAT-URL] Sitemap handling result: ' . $result);
1541	959
1542		- if ($result === 'queued') {
1543		- set_transient('mxchat_admin_notice_success',
1544		- esc_html__('Sitemap queued for processing. Processing will start automatically.', 'mxchat'),
	960	+ if ($result === 'scheduled') {
	961	+ set_transient(
	962	+ 'mxchat_last_sitemap_url',
	963	+ sanitize_text_field($submitted_url),
	964	+ DAY_IN_SECONDS
	965	+ );
	966	+ set_transient('mxchat_admin_notice_info',
	967	+ esc_html__('Sitemap processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1545	968	30
1546	969	);
1547	970	} else {
	971	+ // Return to the admin page without a redirect for better error display
	972	+ // The error is already stored in the sitemap status transient
1548	973	set_transient('mxchat_admin_notice_error',
1549		- esc_html__('Failed to queue sitemap processing. Please check the status below for details.', 'mxchat'),
	974	+ esc_html__('Failed to start sitemap processing. Please check the status below for details.', 'mxchat'),
1550	975	30
1551	976	);
1552	977	}
1553	978
		@@ -1554,50 +979,127 @@
1554	979	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1555	980	exit;
1556	981	}
1557	982
1558		- // Handle Regular URL (single page)
	983	+ // Handle Regular URL
	984	+ //error_log('[MXCHAT-URL] Processing as regular webpage');
1559	985	$page_content = $this->mxchat_extract_main_content($body_content);
	986	+ //error_log('[MXCHAT-URL] Extracted content length: ' . strlen($page_content) . ' bytes');
	987	+
1560	988	$sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	989	+ //error_log('[MXCHAT-URL] Sanitized content length: ' . strlen($sanitized_content) . ' bytes');
1561	990
1562		- //error_log('[MXCHAT-URL-DEBUG] URL: ' . $submitted_url);
1563		- //error_log('[MXCHAT-URL-DEBUG] Extracted page_content length: ' . strlen($page_content));
1564		- //error_log('[MXCHAT-URL-DEBUG] Sanitized content length: ' . strlen($sanitized_content));
1565		- //error_log('[MXCHAT-URL-DEBUG] Content preview: ' . substr($sanitized_content, 0, 500));
1566		-
1567	991	if (empty($sanitized_content)) {
	992	+ //error_log('[MXCHAT-URL] Error: No valid content after sanitization');
	993	+
	994	+ // Set both transients - the error notice and the URL status
1568	995	set_transient('mxchat_admin_notice_error',
1569	996	esc_html__('No valid content found on the provided URL.', 'mxchat'),
1570	997	30
1571	998	);
	999	+
	1000	+ // Set URL status transient
	1001	+ set_transient('mxchat_single_url_status', [
	1002	+ 'url' => $submitted_url,
	1003	+ 'timestamp' => current_time('mysql'),
	1004	+ 'status' => 'failed',
	1005	+ 'error' => esc_html__('No valid content found on the provided URL.', 'mxchat')
	1006	+ ], DAY_IN_SECONDS);
	1007	+
1572	1008	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1573	1009	exit;
1574	1010	}
1575	1011
1576		- // For single URLs, process immediately using submit_content_to_db
1577		- // This handles chunking automatically for large content
1578		- $db_result = MxChat_Utils::submit_content_to_db(
1579		- $sanitized_content,
1580		- $submitted_url,
1581		- $api_key,
1582		- null,
1583		- $bot_id,
1584		- 'url' // content_type
1585		- );
	1012	+ //error_log('[MXCHAT-URL] Generating embedding for content');
	1013	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
1586	1014
1587		- if (is_wp_error($db_result)) {
1588		- $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1015	+ // Check if embedding_vector is a string (error message)
	1016	+ if (is_string($embedding_vector)) {
	1017	+ //error_log('[MXCHAT-URL] Error generating embedding: ' . $embedding_vector);
	1018	+ $error_message = esc_html__('Failed to generate embedding: ', 'mxchat') . esc_html($embedding_vector);
	1019	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1020	+
	1021	+ // Set both transients
1589	1022	set_transient('mxchat_admin_notice_error', $error_message, 30);
1590		- } else {
	1023	+
	1024	+ // Set URL status transient
	1025	+ set_transient('mxchat_single_url_status', [
	1026	+ 'url' => $submitted_url,
	1027	+ 'timestamp' => current_time('mysql'),
	1028	+ 'status' => 'failed',
	1029	+ 'error' => $error_message
	1030	+ ], DAY_IN_SECONDS);
	1031	+
	1032	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1033	+ exit;
	1034	+ }
	1035	+
	1036	+ if (is_array($embedding_vector)) {
	1037	+ //error_log('[MXCHAT-URL] Successfully generated embedding with ' . count($embedding_vector) . ' dimensions');
	1038	+
	1039	+ $db_result = MxChat_Utils::submit_content_to_db(
	1040	+ $sanitized_content,
	1041	+ $submitted_url,
	1042	+ $api_key
	1043	+ );
	1044	+
	1045	+ if (is_wp_error($db_result)) {
	1046	+ //error_log('[MXCHAT-URL] Error: Failed to store content in database: ' . $db_result->get_error_message());
	1047	+ $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1048	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1049	+
	1050	+ // Set both transients
	1051	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1052	+
	1053	+ // Set URL status transient
	1054	+ set_transient('mxchat_single_url_status', [
	1055	+ 'url' => $submitted_url,
	1056	+ 'timestamp' => current_time('mysql'),
	1057	+ 'status' => 'failed',
	1058	+ 'error' => $error_message
	1059	+ ], DAY_IN_SECONDS);
	1060	+
	1061	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1062	+ exit;
	1063	+ }
	1064	+
	1065	+ //error_log('[MXCHAT-URL] Successfully stored content in database');
1591	1066	$success_message = esc_html__('URL content successfully submitted!', 'mxchat');
	1067	+ //error_log('[MXCHAT-URL] Setting success transient: ' . $success_message);
	1068	+
	1069	+ // Set both transients
1592	1070	set_transient('mxchat_admin_notice_success', $success_message, 30);
	1071	+
	1072	+ // Set URL status transient with success
	1073	+ set_transient('mxchat_single_url_status', [
	1074	+ 'url' => $submitted_url,
	1075	+ 'timestamp' => current_time('mysql'),
	1076	+ 'status' => 'complete',
	1077	+ 'content_length' => strlen($sanitized_content),
	1078	+ 'embedding_dimensions' => count($embedding_vector)
	1079	+ ], DAY_IN_SECONDS);
	1080	+
	1081	+ } else {
	1082	+ //error_log('[MXCHAT-URL] Error: Failed to generate embedding. Unexpected result type: ' . gettype($embedding_vector));
	1083	+ $error_message = esc_html__('Failed to generate embedding: Unexpected result type. Please check your API key and try again.', 'mxchat');
	1084	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1085	+
	1086	+ // Set both transients
	1087	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1088	+
	1089	+ // Set URL status transient
	1090	+ set_transient('mxchat_single_url_status', [
	1091	+ 'url' => $submitted_url,
	1092	+ 'timestamp' => current_time('mysql'),
	1093	+ 'status' => 'failed',
	1094	+ 'error' => $error_message
	1095	+ ], DAY_IN_SECONDS);
1593	1096	}
1594	1097
	1098	+ //error_log('[MXCHAT-URL] ===== Completed URL submission process =====');
1595	1099	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1596	1100	exit;
1597	1101	}
1598		-
1599		-
1600	1102	public function mxchat_get_single_url_status() {
1601	1103	$status = get_transient('mxchat_single_url_status');
1602	1104	if (!$status) {
1603	1105	return null;
		@@ -1609,11 +1111,13 @@
1609	1111	}
1610	1112
1611	1113	return $status;
1612	1114	}
1613		-
1614		-public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url, $bot_id = 'default') {
	1115	+public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url) {
	1116	+ // Clear any single URL status when starting sitemap processing
	1117	+ delete_transient('mxchat_single_url_status');
1615	1118	if (!current_user_can('manage_options')) {
	1119	+ //error_log(esc_html__('Unauthorized sitemap processing attempt', 'mxchat'));
1616	1120	return false;
1617	1121	}
1618	1122
1619	1123	try {
		@@ -1622,96 +1126,331 @@
1622	1126	if (!$xml || !is_object($xml)) {
1623	1127	throw new Exception(__('Invalid XML object provided', 'mxchat'));
1624	1128	}
1625	1129
1626		- // Get bot-specific embedding API for validation
1627		- $bot_options = $this->get_bot_options($bot_id);
1628		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	1130	+ // Add embedding validation before processing
	1131	+ // Test embedding with a small sample text to verify API key is working
	1132	+ $test_result = $this->mxchat_generate_embedding("This is a test to verify the embedding API key is working.");
1629	1133
1630		- // Test the embedding API before processing
1631		- $test_phrase = "Test embedding generation for MxChat";
1632		- $test_result = $this->mxchat_generate_embedding($test_phrase, $bot_id);
1633		-
	1134	+ // Check if test_result is a string (error message) rather than an array (valid embedding)
1634	1135	if (is_string($test_result)) {
	1136	+ //error_log('[MXCHAT-URL] Embedding API validation failed: ' . $test_result);
	1137	+
	1138	+ // Store the error in the status transient so it can be displayed later
	1139	+ $status_data = array(
	1140	+ 'total_urls' => 0,
	1141	+ 'processed_urls' => 0,
	1142	+ 'status' => 'error',
	1143	+ 'error' => __('Embedding API validation failed: ', 'mxchat') . $test_result,
	1144	+ 'last_update' => time()
	1145	+ );
	1146	+
	1147	+ set_transient(
	1148	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1149	+ array_map('sanitize_text_field', $status_data),
	1150	+ DAY_IN_SECONDS
	1151	+ );
	1152	+
1635	1153	throw new Exception(__('Embedding API validation failed: ', 'mxchat') . $test_result);
1636	1154	}
1637	1155
	1156	+ // Make sure it's an array (valid embedding)
1638	1157	if (!is_array($test_result)) {
	1158	+ //error_log('[MXCHAT-URL] Embedding API returned unexpected result type: ' . gettype($test_result));
1639	1159	throw new Exception(__('Embedding API returned unexpected result type. Please check your configuration.', 'mxchat'));
1640	1160	}
1641	1161
1642		- // Extract URLs from sitemap
1643		- $urls = array();
	1162	+ $urls = [];
1644	1163	foreach ($xml->url as $url_element) {
1645	1164	$url = esc_url_raw((string)$url_element->loc);
1646	1165	if ($url) {
1647		- $urls[] = array('url' => $url);
	1166	+ $urls[] = $url;
1648	1167	}
1649	1168	}
1650	1169
1651		- $total_urls = count($urls);
	1170	+ $total_urls = absint(count($urls));
1652	1171
1653	1172	if ($total_urls < 1) {
1654	1173	throw new Exception(__('No valid URLs found in sitemap', 'mxchat'));
1655	1174	}
1656	1175
1657		- // Create unique queue ID
1658		- $queue_id = 'sitemap_' . md5($sitemap_url . time());
	1176	+ wp_schedule_single_event(time(), 'mxchat_process_sitemap_urls', array(
	1177	+ 'urls' => $urls,
	1178	+ 'sitemap_url' => $sitemap_url,
	1179	+ 'total_urls' => $total_urls,
	1180	+ 'batch_size' => absint(10),
	1181	+ 'batch_pause' => absint(5)
	1182	+ ));
1659	1183
1660		- // Add URLs to queue
1661		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'url', $urls, $bot_id);
	1184	+ $status_data = array(
	1185	+ 'total_urls' => $total_urls,
	1186	+ 'processed_urls' => 0,
	1187	+ 'status' => 'processing',
	1188	+ 'last_update' => time()
	1189	+ );
1662	1190
1663		- if ($queued_count === 0) {
1664		- throw new Exception(__('Failed to add URLs to processing queue', 'mxchat'));
1665		- }
	1191	+ set_transient(
	1192	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1193	+ array_map('sanitize_text_field', $status_data),
	1194	+ DAY_IN_SECONDS
	1195	+ );
1666	1196
1667		- // Store queue metadata
1668		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $sitemap_url);
1669		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'sitemap');
1670		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_urls);
1671		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
1672		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	1197	+ return __('scheduled', 'mxchat');
1673	1198
1674		- // Store queue ID in transient for status tracking
1675		- set_transient('mxchat_active_queue_sitemap', $queue_id, DAY_IN_SECONDS);
1676		- set_transient('mxchat_last_sitemap_url', $sitemap_url, DAY_IN_SECONDS);
1677		-
1678		- return 'queued';
1679		-
1680		- } catch (Exception $e) {
	1199	+ } catch (\Exception $e) {
1681	1200	$error_message = $e->getMessage();
1682	1201	//error_log(sprintf(esc_html__('Error preparing sitemap for processing: %s', 'mxchat'), esc_html($error_message)));
1683	1202
	1203	+ // Store the sitemap URL and error in transients so they can be displayed
	1204	+ set_transient(
	1205	+ 'mxchat_last_sitemap_url',
	1206	+ sanitize_text_field($sitemap_url),
	1207	+ DAY_IN_SECONDS
	1208	+ );
	1209	+
	1210	+ $status_data = array(
	1211	+ 'total_urls' => 0,
	1212	+ 'processed_urls' => 0,
	1213	+ 'status' => 'error',
	1214	+ 'error' => $error_message,
	1215	+ 'last_update' => time()
	1216	+ );
	1217	+
	1218	+ set_transient(
	1219	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1220	+ array_map('sanitize_text_field', $status_data),
	1221	+ DAY_IN_SECONDS
	1222	+ );
	1223	+
1684	1224	return $error_message;
1685	1225	}
1686		-
1687	1226	}
1688	1227
1689		-/**
1690		- * Remove shortcode tags but preserve the content inside them
1691		- * Example: [vc_column]Hello World[/vc_column] becomes "Hello World"
1692		- *
1693		- * @param string $content The content containing shortcodes
1694		- * @return string Content with shortcode tags removed but inner content preserved
1695		- */
1696		-private function strip_shortcode_tags_preserve_content($content) {
1697		- // Single-pass regex removes all shortcode brackets: [tag], [tag attr="val"], [/tag], [tag /]
1698		- // Content between tags is inherently preserved since only brackets are targeted
1699		- $result = preg_replace('/\[\/?\w[\w-]*[^\]]*\]/', '', $content);
1700		- return ($result !== null) ? $result : $content;
	1228	+public function mxchat_process_sitemap_urls_cron($urls, $sitemap_url, $total_urls, $batch_size, $batch_pause) {
	1229	+ // Validate inputs
	1230	+ $sitemap_url = esc_url_raw($sitemap_url);
	1231	+ $total_urls = absint($total_urls);
	1232	+ $batch_size = absint($batch_size);
	1233	+ $batch_pause = absint($batch_pause);
	1234	+
	1235	+ if (!is_array($urls) || empty($urls)) {
	1236	+ return;
	1237	+ }
	1238	+
	1239	+ try {
	1240	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	1241	+ $status = get_transient($status_key);
	1242	+
	1243	+ if (!$status || !is_array($status)) {
	1244	+ throw new Exception('Invalid status data retrieved from transient');
	1245	+ }
	1246	+
	1247	+ // Initialize failed_urls array if it doesn't exist
	1248	+ if (!isset($status['failed_urls_list']) || !is_array($status['failed_urls_list'])) {
	1249	+ $status['failed_urls_list'] = [];
	1250	+ }
	1251	+
	1252	+ $start_url = absint($status['processed_urls']);
	1253	+ $end_url = min($start_url + $batch_size, $total_urls);
	1254	+
	1255	+ // Track batch statistics
	1256	+ $batch_stats = [
	1257	+ 'processed' => 0,
	1258	+ 'failed' => 0,
	1259	+ 'last_error' => '',
	1260	+ 'embedding_errors' => 0
	1261	+ ];
	1262	+
	1263	+ // Check embedding configuration with first URL (only on first batch)
	1264	+ if ($start_url === 0) {
	1265	+ $test_url = esc_url_raw($urls[0]);
	1266	+ $test_response = wp_remote_get($test_url);
	1267	+
	1268	+ if (!is_wp_error($test_response) && wp_remote_retrieve_response_code($test_response) === 200) {
	1269	+ $test_html = wp_remote_retrieve_body($test_response);
	1270	+ $test_content = $this->mxchat_extract_main_content($test_html);
	1271	+ $test_sanitized = $this->mxchat_sanitize_content_for_api($test_content);
	1272	+
	1273	+ if (!empty($test_sanitized)) {
	1274	+ $test_embedding = $this->mxchat_generate_embedding($test_sanitized);
	1275	+
	1276	+ if (is_string($test_embedding)) {
	1277	+ throw new Exception('Embedding generation failed: ' . $test_embedding);
	1278	+ }
	1279	+
	1280	+ if (!is_array($test_embedding)) {
	1281	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($test_embedding));
	1282	+ }
	1283	+ }
	1284	+ }
	1285	+ }
	1286	+
	1287	+ for ($i = $start_url; $i < $end_url; $i++) {
	1288	+ $page_url = esc_url_raw($urls[$i]);
	1289	+ $max_retries = 3;
	1290	+ $retry_count = 0;
	1291	+ $url_processed = false;
	1292	+ $last_error = '';
	1293	+
	1294	+ while (!$url_processed && $retry_count < $max_retries) {
	1295	+ try {
	1296	+ // Attempt to fetch the URL
	1297	+ $page_response = wp_remote_get($page_url, array('timeout' => 30));
	1298	+
	1299	+ if (is_wp_error($page_response)) {
	1300	+ throw new Exception('HTTP request failed: ' . $page_response->get_error_message());
	1301	+ }
	1302	+
	1303	+ $response_code = wp_remote_retrieve_response_code($page_response);
	1304	+ if ($response_code !== 200) {
	1305	+ throw new Exception('HTTP Status: ' . $response_code);
	1306	+ }
	1307	+
	1308	+ $page_html = wp_remote_retrieve_body($page_response);
	1309	+
	1310	+ if (empty($page_html)) {
	1311	+ throw new Exception('Empty response body');
	1312	+ }
	1313	+
	1314	+ $page_content = $this->mxchat_extract_main_content($page_html);
	1315	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	1316	+
	1317	+ if (empty($sanitized_content)) {
	1318	+ throw new Exception('No valid content found after processing');
	1319	+ }
	1320	+
	1321	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	1322	+
	1323	+ if (is_string($embedding_vector)) {
	1324	+ throw new Exception('Embedding generation failed: ' . $embedding_vector);
	1325	+ }
	1326	+
	1327	+ if (!is_array($embedding_vector)) {
	1328	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($embedding_vector));
	1329	+ }
	1330	+
	1331	+ // Submit to database
	1332	+ $options = get_option('mxchat_options');
	1333	+ $submission_result = MxChat_Utils::submit_content_to_db($sanitized_content, $page_url, $options['api_key']);
	1334	+
	1335	+ if (is_wp_error($submission_result)) {
	1336	+ throw new Exception('Database submission failed: ' . $submission_result->get_error_message());
	1337	+ }
	1338	+
	1339	+ // Success!
	1340	+ $url_processed = true;
	1341	+ $batch_stats['processed']++;
	1342	+
	1343	+ } catch (Exception $e) {
	1344	+ $retry_count++;
	1345	+ $last_error = $e->getMessage();
	1346	+
	1347	+ //error_log("URL {$page_url} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	1348	+
	1349	+ // Track embedding errors specifically
	1350	+ if (strpos($last_error, 'Embedding') !== false) {
	1351	+ $batch_stats['embedding_errors']++;
	1352	+ }
	1353	+
	1354	+ if ($retry_count < $max_retries) {
	1355	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	1356	+ sleep(pow(2, $retry_count - 1));
	1357	+ }
	1358	+ }
	1359	+ }
	1360	+
	1361	+ // If URL still not processed after all retries, mark as failed
	1362	+ if (!$url_processed) {
	1363	+ $batch_stats['failed']++;
	1364	+ $batch_stats['last_error'] = $last_error;
	1365	+
	1366	+ // Add to failed URLs list
	1367	+ $status['failed_urls_list'][] = [
	1368	+ 'url' => $page_url,
	1369	+ 'error' => $last_error,
	1370	+ 'time' => time(),
	1371	+ 'retries' => $max_retries
	1372	+ ];
	1373	+
	1374	+ // Limit the number of failed URLs we store to prevent transient size issues
	1375	+ if (count($status['failed_urls_list']) > 100) {
	1376	+ $status['failed_urls_list'] = array_slice($status['failed_urls_list'], -100);
	1377	+ }
	1378	+ }
	1379	+
	1380	+ // Update progress
	1381	+ $status['processed_urls'] = absint($i + 1);
	1382	+ $status['last_update'] = time();
	1383	+ $status['failed_urls'] = absint($status['failed_urls'] ?? 0) + ($url_processed ? 0 : 1);
	1384	+ $status['last_error'] = $batch_stats['last_error'];
	1385	+
	1386	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1387	+
	1388	+ // If we have too many consecutive embedding errors, stop processing
	1389	+ if ($batch_stats['embedding_errors'] >= 10) {
	1390	+ throw new Exception('Too many consecutive embedding failures detected. Please check your API configuration.');
	1391	+ }
	1392	+ }
	1393	+
	1394	+ // If all URLs in this batch failed, stop processing
	1395	+ if ($batch_stats['processed'] === 0 && $batch_stats['failed'] > 0) {
	1396	+ $status['status'] = 'error';
	1397	+ $status['error'] = sprintf(
	1398	+ 'Processing stopped: %d consecutive failures in batch. Last error: %s',
	1399	+ $batch_stats['failed'],
	1400	+ $batch_stats['last_error']
	1401	+ );
	1402	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1403	+ return;
	1404	+ }
	1405	+
	1406	+ // Update final progress
	1407	+ $status['processed_urls'] = min($end_url, $total_urls);
	1408	+ $status['last_update'] = time();
	1409	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1410	+
	1411	+ // Check if we've processed all URLs
	1412	+ if ($end_url >= $total_urls) {
	1413	+ // All URLs have been processed - mark as complete
	1414	+ $status['status'] = 'complete';
	1415	+ $status['processed_urls'] = $total_urls;
	1416	+
	1417	+ // Add completion summary
	1418	+ $status['completion_summary'] = [
	1419	+ 'total_urls' => $total_urls,
	1420	+ 'successful_urls' => $total_urls - absint($status['failed_urls'] ?? 0),
	1421	+ 'failed_urls' => absint($status['failed_urls'] ?? 0),
	1422	+ 'completion_time' => current_time('mysql')
	1423	+ ];
	1424	+
	1425	+ // Save the completed status (don't delete it - let user dismiss manually)
	1426	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1427	+
	1428	+ // DON'T delete the status transients here - let user dismiss manually
	1429	+ } else {
	1430	+ // Schedule next batch
	1431	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_sitemap_urls', array(
	1432	+ 'urls' => $urls,
	1433	+ 'sitemap_url' => $sitemap_url,
	1434	+ 'total_urls' => $total_urls,
	1435	+ 'batch_size' => $batch_size,
	1436	+ 'batch_pause' => $batch_pause,
	1437	+ ));
	1438	+ }
	1439	+ } catch (\Exception $e) {
	1440	+ $status['status'] = 'error';
	1441	+ $status['error'] = $e->getMessage();
	1442	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1443	+ }
1701	1444	}
1702		-
1703	1445	public function mxchat_sanitize_content_for_api($content) {
1704	1446	//error_log('[MXCHAT-SANITIZE] Original content preview: ' . substr($content, 0, 500) . '...');
1705		-
1706		- // Remove shortcode tags but PRESERVE content inside them
1707		- $content = $this->strip_shortcode_tags_preserve_content($content);
1708		-
	1447	+
1709	1448	// Remove script, style tags, and HTML comments
1710	1449	$content = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $content);
1711	1450	$content = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $content);
1712	1451	$content = preg_replace('/<!--(.|\s)*?-->/', '', $content);
1713		-
	1452	+
1714	1453	// Remove all HTML tags and decode HTML entities
1715	1454	$content = wp_strip_all_tags($content);
1716	1455	$content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5);
1717	1456
		@@ -1769,12 +1508,12 @@
1769	1508	@$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'), LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
1770	1509	$xpath = new DOMXPath($dom);
1771	1510
1772	1511	// For debugging purposes
1773		- $debugEnabled = true; // Set to true to enable debugging output
	1512	+ $debugEnabled = false; // Set to true to enable debugging output
1774	1513	$debug = function($message) use ($debugEnabled) {
1775	1514	if ($debugEnabled) {
1776		- //error_log('[MXCHAT-EXTRACT-DEBUG] ' . $message);
	1515	+ //error_log('[MXCHAT-DEBUG] ' . $message);
1777	1516	}
1778	1517	};
1779	1518
1780	1519	// Direct targeting for Gerow theme posts
		@@ -1873,25 +1612,18 @@
1873	1612	'//main',
1874	1613	'//div[contains(@class, "content")]'
1875	1614	];
1876	1615
1877		- // First handle Elementor content - get only leaf widget containers to avoid duplicates
	1616	+ // First handle Elementor content
1878	1617	$debug("Checking for Elementor content");
1879		- // Get widget containers that are direct children of widgets (not nested inside other widget containers)
1880		- $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-widget")]//div[contains(@class, "elementor-widget-container")]');
	1618	+ $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-element")]//div[contains(@class, "elementor-widget-container")]');
1881	1619	if ($elementor_widgets && $elementor_widgets->length > 0) {
1882	1620	$debug("Found Elementor widgets");
1883		- $seen_content = array(); // Track seen content to avoid duplicates
1884	1621	$combined_content = '';
1885	1622	foreach ($elementor_widgets as $widget) {
1886	1623	$widget_content = $dom->saveHTML($widget);
1887	1624	if (!empty($widget_content)) {
1888		- // Create a hash of the content to detect duplicates
1889		- $content_hash = md5($widget_content);
1890		- if (!isset($seen_content[$content_hash])) {
1891		- $seen_content[$content_hash] = true;
1892		- $combined_content .= $widget_content;
1893		- }
	1625	+ $combined_content .= $widget_content;
1894	1626	}
1895	1627	}
1896	1628	if (!empty($combined_content)) {
1897	1629	$debug("Returning Elementor content");
		@@ -1903,14 +1635,15 @@
1903	1635	foreach ($selectors as $selector) {
1904	1636	$debug("Trying selector: " . $selector);
1905	1637	$nodes = $xpath->query($selector);
1906	1638	if ($nodes && $nodes->length > 0) {
1907		- $debug("Found " . $nodes->length . " matches for selector: " . $selector);
1908		- // Only take the FIRST matching node to avoid duplicate content
1909		- // (pages often have nested or multiple containers with same class)
1910		- $content = $dom->saveHTML($nodes->item(0));
	1639	+ $debug("Found matches for selector: " . $selector);
	1640	+ $content = '';
	1641	+ foreach ($nodes as $node) {
	1642	+ $content .= $dom->saveHTML($node);
	1643	+ }
1911	1644	if (!empty($content)) {
1912		- $debug("Returning content from selector: " . $selector . " (first match only)");
	1645	+ $debug("Returning content from selector: " . $selector);
1913	1646	return $content;
1914	1647	}
1915	1648	}
1916	1649	}
		@@ -1934,106 +1667,16 @@
1934	1667	$debug("Returning blog-area section content");
1935	1668	return $content;
1936	1669	}
1937	1670	}
1938		-
1939		- // Generic container selectors for non-CMS sites (like .asp pages)
1940		- $debug("Trying generic container selectors");
1941		- $generic_selectors = [
1942		- '//div[@id="main"]',
1943		- '//div[@id="wrapper"]',
1944		- '//div[@id="page"]',
1945		- '//div[@id="site-content"]',
1946		- '//div[contains(@class, "main-content")]',
1947		- '//div[contains(@class, "page-content")]',
1948		- '//div[contains(@class, "site-content")]',
1949		- ];
1950		-
1951		- foreach ($generic_selectors as $selector) {
1952		- $debug("Trying generic selector: " . $selector);
1953		- $nodes = $xpath->query($selector);
1954		- if ($nodes && $nodes->length > 0) {
1955		- $content = $dom->saveHTML($nodes->item(0));
1956		- if (!empty($content)) {
1957		- $debug("Returning content from generic selector: " . $selector);
1958		- return $content;
1959		- }
1960		- }
1961		- }
1962		-
1963		- // Paragraph-based content detection - find regions with substantial text
1964		- $debug("Trying paragraph-based content detection");
1965		- $paragraphs = $xpath->query('//p[string-length(normalize-space()) > 50]');
1966		- if ($paragraphs && $paragraphs->length >= 3) {
1967		- $debug("Found " . $paragraphs->length . " substantial paragraphs");
1968		- // Collect all substantial paragraphs and their content
1969		- $paragraph_content = '';
1970		- foreach ($paragraphs as $p) {
1971		- $paragraph_content .= $dom->saveHTML($p) . "\n";
1972		- }
1973		- if (!empty($paragraph_content)) {
1974		- $debug("Returning paragraph-based content");
1975		- return $paragraph_content;
1976		- }
1977		- }
1978		-
1979		- // Improved body fallback - strip nav/header/footer elements first
1980		- $debug("Using improved body fallback");
	1671	+
	1672	+ // Fallback: Return the body content if no specific selector matches
	1673	+ $debug("Using body fallback");
1981	1674	$body = $dom->getElementsByTagName('body');
1982	1675	if ($body->length > 0) {
1983		- // Clone the body to avoid modifying the original DOM
1984		- $body_clone = $body->item(0)->cloneNode(true);
1985		-
1986		- // Remove common non-content elements by tag name
1987		- $remove_tags = ['nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript'];
1988		- foreach ($remove_tags as $tag) {
1989		- $elements = $body_clone->getElementsByTagName($tag);
1990		- // Iterate backwards to safely remove elements
1991		- for ($i = $elements->length - 1; $i >= 0; $i--) {
1992		- $el = $elements->item($i);
1993		- if ($el && $el->parentNode) {
1994		- $el->parentNode->removeChild($el);
1995		- }
1996		- }
1997		- }
1998		-
1999		- // Remove elements with common non-content class names using XPath on the cloned body
2000		- $temp_dom = new DOMDocument();
2001		- @$temp_dom->appendChild($temp_dom->importNode($body_clone, true));
2002		- $temp_xpath = new DOMXPath($temp_dom);
2003		-
2004		- $remove_class_patterns = [
2005		- '//*[contains(@class, "nav")]',
2006		- '//*[contains(@class, "menu")]',
2007		- '//*[contains(@class, "sidebar")]',
2008		- '//*[contains(@class, "footer")]',
2009		- '//*[contains(@class, "header")]',
2010		- '//*[contains(@id, "nav")]',
2011		- '//*[contains(@id, "menu")]',
2012		- '//*[contains(@id, "sidebar")]',
2013		- '//*[contains(@id, "footer")]',
2014		- '//*[contains(@id, "header")]',
2015		- ];
2016		-
2017		- foreach ($remove_class_patterns as $pattern) {
2018		- $elements = $temp_xpath->query($pattern);
2019		- if ($elements) {
2020		- for ($i = $elements->length - 1; $i >= 0; $i--) {
2021		- $el = $elements->item($i);
2022		- if ($el && $el->parentNode) {
2023		- $el->parentNode->removeChild($el);
2024		- }
2025		- }
2026		- }
2027		- }
2028		-
2029		- $cleaned_content = $temp_dom->saveHTML();
2030		- if (!empty($cleaned_content)) {
2031		- $debug("Returning cleaned body content");
2032		- return $cleaned_content;
2033		- }
	1676	+ return $dom->saveHTML($body->item(0));
2034	1677	}
2035		-
	1678	+
2036	1679	// Last resort: return the original HTML
2037	1680	$debug("Returning original HTML");
2038	1681	return $html;
2039	1682	} catch (Exception $e) {
		@@ -2085,41 +1728,47 @@
2085	1728	try {
2086	1729	// Verify the request
2087	1730	check_ajax_referer('mxchat_status_nonce', 'nonce');
2088	1731
2089		- // Get active queue IDs
2090		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
2091		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
	1732	+ // Get the status just like in your admin page
	1733	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1734	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
2092	1735
2093		- $sitemap_status = false;
2094		- $pdf_status = false;
	1736	+ $pdf_status = $pdf_url ? $this->mxchat_get_pdf_processing_status($pdf_url) : false;
	1737	+ $sitemap_status = $sitemap_url ? $this->mxchat_get_sitemap_processing_status($sitemap_url) : false;
2095	1738
2096		- // Get sitemap queue status
2097		- if ($sitemap_queue_id) {
2098		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
	1739	+ // Add the PDF URL to the status object
	1740	+ if ($pdf_status && $pdf_url) {
	1741	+ $pdf_status['pdf_url'] = $pdf_url;
2099	1742	}
2100	1743
2101		- // Get PDF queue status
2102		- if ($pdf_queue_id) {
2103		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
2104		- }
	1744	+ // Set the current PDF URL for the manual batch processing button
	1745	+ $current_pdf_url = $pdf_url;
2105	1746
	1747	+ // Check for true processing status, not just presence of status
2106	1748	$is_active_processing =
2107		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
2108		- ($pdf_status && $pdf_status['status'] === 'processing');
	1749	+ ($sitemap_status && isset($sitemap_status['status']) && $sitemap_status['status'] === 'processing') ||
	1750	+ ($pdf_status && isset($pdf_status['status']) && $pdf_status['status'] === 'processing');
2109	1751
	1752	+ // Get single URL status, but only if no processing is active
	1753	+ $single_url_status = !$is_active_processing ? $this->mxchat_get_single_url_status() : false;
	1754	+
	1755	+ // REMOVED: Auto-clearing of completed status - now only done via dismiss button
	1756	+
2110	1757	// Return JSON response with the status data
2111	1758	wp_send_json(array(
2112	1759	'pdf_status' => $pdf_status,
2113	1760	'sitemap_status' => $sitemap_status,
	1761	+ 'single_url_status' => $single_url_status,
2114	1762	'is_processing' => $is_active_processing,
2115		- 'sitemap_queue_id' => $sitemap_queue_id,
2116		- 'pdf_queue_id' => $pdf_queue_id
	1763	+ 'current_pdf_url' => $current_pdf_url
2117	1764	));
2118	1765
2119	1766	} catch (Exception $e) {
	1767	+ // Log the error
2120	1768	//error_log('MxChat Status Update Error: ' . $e->getMessage());
2121	1769
	1770	+ // Return a friendly error response
2122	1771	wp_send_json_error(array(
2123	1772	'message' => 'Error getting status updates: ' . $e->getMessage(),
2124	1773	'status' => 'error'
2125	1774	));
		@@ -2124,1371 +1773,8 @@
2124	1773	'status' => 'error'
2125	1774	));
2126	1775	}
2127	1776	}
2128		-
2129		-/**
2130		- * Helper function to get queue status data
2131		- */
2132		-private function mxchat_get_queue_status_data($queue_id, $type = 'sitemap') {
2133		- global $wpdb;
2134		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
2135		-
2136		- // Get counts by status
2137		- $counts = $wpdb->get_results($wpdb->prepare(
2138		- "SELECT status, COUNT(*) as count
2139		- FROM $table_name
2140		- WHERE queue_id = %s
2141		- GROUP BY status",
2142		- $queue_id
2143		- ), OBJECT_K);
2144		-
2145		- $total = 0;
2146		- $completed = 0;
2147		- $failed = 0;
2148		- $processing = 0;
2149		- $pending = 0;
2150		-
2151		- foreach ($counts as $status => $data) {
2152		- $count = absint($data->count);
2153		- $total += $count;
2154		-
2155		- switch ($status) {
2156		- case 'completed':
2157		- $completed = $count;
2158		- break;
2159		- case 'failed':
2160		- $failed = $count;
2161		- break;
2162		- case 'processing':
2163		- $processing = $count;
2164		- break;
2165		- case 'pending':
2166		- $pending = $count;
2167		- break;
2168		- }
2169		- }
2170		-
2171		- if ($total === 0) {
2172		- return false;
2173		- }
2174		-
2175		- // Calculate percentage
2176		- $percentage = round((($completed + $failed) / $total) * 100);
2177		-
2178		- // Get failed items details (limit to 50)
2179		- $failed_items = array();
2180		- if ($failed > 0) {
2181		- $failed_results = $wpdb->get_results($wpdb->prepare(
2182		- "SELECT item_type, item_data, error_message, attempts, completed_at
2183		- FROM $table_name
2184		- WHERE queue_id = %s
2185		- AND status = 'failed'
2186		- AND attempts >= max_attempts
2187		- ORDER BY id DESC
2188		- LIMIT 50",
2189		- $queue_id
2190		- ));
2191		-
2192		- foreach ($failed_results as $item) {
2193		- $data = json_decode($item->item_data, true);
2194		- $url = $type === 'pdf' ? ($data['pdf_url'] ?? '') : ($data['url'] ?? '');
2195		-
2196		- $failed_items[] = array(
2197		- 'url' => $url,
2198		- 'page' => $type === 'pdf' ? ($data['page_number'] ?? 0) : 0,
2199		- 'error' => $item->error_message,
2200		- 'retries' => $item->attempts,
2201		- 'time' => strtotime($item->completed_at)
2202		- );
2203		- }
2204		- }
2205		-
2206		- // Get queue metadata
2207		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
2208		-
2209		- // Determine if queue is complete
2210		- $is_complete = ($pending === 0 && $processing === 0);
2211		-
2212		- // Get last update time
2213		- $last_update = $wpdb->get_var($wpdb->prepare(
2214		- "SELECT MAX(UNIX_TIMESTAMP(COALESCE(completed_at, started_at, created_at)))
2215		- FROM $table_name
2216		- WHERE queue_id = %s",
2217		- $queue_id
2218		- ));
2219		-
2220		- $last_update_text = $last_update ? human_time_diff($last_update, time()) . ' ' . __('ago', 'mxchat') : __('Just now', 'mxchat');
2221		-
2222		- // Format based on type
2223		- if ($type === 'pdf') {
2224		- return array(
2225		- 'total_pages' => $total,
2226		- 'processed_pages' => $completed + $failed,
2227		- 'failed_pages' => $failed,
2228		- 'percentage' => $percentage,
2229		- 'status' => $is_complete ? 'complete' : 'processing',
2230		- 'last_update' => $last_update_text,
2231		- 'failed_pages_list' => $failed_items,
2232		- 'pdf_url' => $source_url,
2233		- 'queue_id' => $queue_id
2234		- );
2235		- } else {
2236		- return array(
2237		- 'total_urls' => $total,
2238		- 'processed_urls' => $completed + $failed,
2239		- 'failed_urls' => $failed,
2240		- 'percentage' => $percentage,
2241		- 'status' => $is_complete ? 'complete' : 'processing',
2242		- 'last_update' => $last_update_text,
2243		- 'failed_urls_list' => $failed_items,
2244		- 'sitemap_url' => $source_url,
2245		- 'queue_id' => $queue_id
2246		- );
2247		- }
2248		-}
2249		-
2250		-/**
2251		- * Public method to get processing status for both sitemap and PDF queues
2252		- * Used by admin pages to display processing status
2253		- *
2254		- * @return array Array with 'sitemap_status', 'pdf_status', and 'is_processing' keys
2255		- */
2256		-public function mxchat_get_processing_statuses() {
2257		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
2258		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
2259		-
2260		- $sitemap_status = false;
2261		- $pdf_status = false;
2262		-
2263		- if ($sitemap_queue_id) {
2264		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
2265		- }
2266		-
2267		- if ($pdf_queue_id) {
2268		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
2269		- }
2270		-
2271		- $is_processing =
2272		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
2273		- ($pdf_status && $pdf_status['status'] === 'processing');
2274		-
2275		- return array(
2276		- 'sitemap_status' => $sitemap_status,
2277		- 'pdf_status' => $pdf_status,
2278		- 'is_processing' => $is_processing
2279		- );
2280		-}
2281		-
2282		-/**
2283		- * AJAX handler to get recent knowledge entries for real-time table updates
2284		- * UPDATED: Now supports both WordPress DB and Pinecone data sources
2285		- */
2286		-public function ajax_mxchat_get_recent_entries() {
2287		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2288		-
2289		- if (!current_user_can('manage_options')) {
2290		- wp_send_json_error(array('message' => 'Unauthorized'));
2291		- return;
2292		- }
2293		-
2294		- global $wpdb;
2295		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2296		-
2297		- // Get parameters
2298		- $last_id = isset($_POST['last_id']) ? absint($_POST['last_id']) : 0;
2299		- $limit = isset($_POST['limit']) ? min(absint($_POST['limit']), 50) : 10;
2300		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2301		-
2302		- // Check if Pinecone is enabled for this bot
2303		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
2304		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
2305		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2306		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
2307		-
2308		- if ($use_pinecone && $has_pinecone_api) {
2309		- // PINECONE DATA SOURCE - Get grouped entry count (not raw vector count)
2310		- // Use mxchat_fetch_pinecone_records which returns total_unique_entries
2311		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, '', 1, 10, $bot_id, '');
2312		- $total_count = $records['total'] ?? 0;
2313		-
2314		- // For Pinecone, we don't return individual entries during polling
2315		- // (entries are already displayed on page load via mxchat_fetch_pinecone_records)
2316		- // We just return the updated count
2317		- wp_send_json_success(array(
2318		- 'entries' => array(),
2319		- 'total_count' => absint($total_count),
2320		- 'max_id' => $last_id,
2321		- 'data_source' => 'pinecone'
2322		- ));
2323		- return;
2324		- }
2325		-
2326		- // WORDPRESS DB DATA SOURCE
2327		- // Build query to get entries newer than last_id
2328		- $where_clauses = array('1=1');
2329		- $where_values = array();
2330		-
2331		- if ($last_id > 0) {
2332		- $where_clauses[] = 'id > %d';
2333		- $where_values[] = $last_id;
2334		- }
2335		-
2336		- // Note: WordPress DB table doesn't have bot_id column
2337		- // Multi-bot filtering is handled via Pinecone namespaces
2338		-
2339		- $where_sql = implode(' AND ', $where_clauses);
2340		-
2341		- // Get recent entries
2342		- $query = "SELECT id, article_content, source_url, timestamp
2343		- FROM $table_name
2344		- WHERE $where_sql
2345		- ORDER BY id DESC
2346		- LIMIT %d";
2347		-
2348		- $where_values[] = $limit;
2349		-
2350		- $entries = $wpdb->get_results($wpdb->prepare($query, $where_values));
2351		-
2352		- // Get total count of GROUPED entries (by source_url) - matches pagination display
2353		- // Count unique source_urls (excluding mxchat:// internal refs) + count of ungrouped rows
2354		- $total_count = $wpdb->get_var(
2355		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2356		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
2357		- );
2358		-
2359		- // Format entries for response
2360		- $formatted_entries = array();
2361		- $preview_length = 150;
2362		- foreach ($entries as $entry) {
2363		- // Parse chunk metadata using the proper chunker method (same as initial page load)
2364		- if (class_exists('MxChat_Chunker')) {
2365		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($entry->article_content);
2366		- $display_content = $chunk_meta['text'];
2367		- $chunk_metadata = $chunk_meta['metadata'];
2368		- } else {
2369		- $display_content = $entry->article_content;
2370		- $chunk_metadata = array();
2371		- }
2372		-
2373		- $content_preview = mb_strlen($display_content) > $preview_length
2374		- ? mb_substr($display_content, 0, $preview_length) . '...'
2375		- : $display_content;
2376		-
2377		- $formatted_entries[] = array(
2378		- 'id' => $entry->id,
2379		- 'preview' => esc_html($content_preview),
2380		- 'full_content' => wp_kses_post(wpautop($display_content)),
2381		- 'content_length' => mb_strlen($display_content),
2382		- 'preview_length' => $preview_length,
2383		- 'source_url' => $entry->source_url,
2384		- 'has_link' => !empty($entry->source_url) && strpos($entry->source_url, 'mxchat://') !== 0,
2385		- 'chunk_metadata' => $chunk_metadata,
2386		- 'bot_id' => $entry->bot_id ?? 'default',
2387		- 'edit_nonce' => wp_create_nonce('mxchat_edit_entry_nonce'),
2388		- 'delete_nonce' => wp_create_nonce('mxchat_delete_prompt_nonce')
2389		- );
2390		- }
2391		-
2392		- wp_send_json_success(array(
2393		- 'entries' => $formatted_entries,
2394		- 'total_count' => absint($total_count),
2395		- 'max_id' => !empty($entries) ? $entries[0]->id : $last_id,
2396		- 'data_source' => 'wordpress'
2397		- ));
2398		-}
2399		-
2400		-/**
2401		- * Get Pinecone total count from stats API
2402		- * Helper function for ajax_mxchat_get_recent_entries
2403		- */
2404		-private function mxchat_get_pinecone_count_from_stats($pinecone_options) {
2405		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2406		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
2407		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
2408		-
2409		- if (empty($api_key) || empty($host)) {
2410		- return 0;
2411		- }
2412		-
2413		- try {
2414		- $stats_url = "https://{$host}/describe_index_stats";
2415		-
2416		- $response = wp_remote_post($stats_url, array(
2417		- 'headers' => array(
2418		- 'Api-Key' => $api_key,
2419		- 'Content-Type' => 'application/json'
2420		- ),
2421		- 'body' => '{}',
2422		- 'timeout' => 10
2423		- ));
2424		-
2425		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2426		- $body = wp_remote_retrieve_body($response);
2427		- $stats_data = json_decode($body, true);
2428		-
2429		- // If namespace is specified, get count from that specific namespace
2430		- if (!empty($namespace) && isset($stats_data['namespaces'][$namespace]['vectorCount'])) {
2431		- return intval($stats_data['namespaces'][$namespace]['vectorCount']);
2432		- }
2433		-
2434		- // If no namespace specified or namespace not found in response, use total
2435		- return intval($stats_data['totalVectorCount'] ?? 0);
2436		- }
2437		-
2438		- return 0;
2439		-
2440		- } catch (Exception $e) {
2441		- return 0;
2442		- }
2443		-}
2444		-
2445		-/**
2446		- * AJAX handler to refresh Pinecone entries table via AJAX
2447		- * Returns the table HTML for updating the UI without a full page reload
2448		- */
2449		-public function ajax_mxchat_refresh_pinecone_entries() {
2450		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2451		-
2452		- if (!current_user_can('manage_options')) {
2453		- wp_send_json_error(array('message' => 'Unauthorized'));
2454		- return;
2455		- }
2456		-
2457		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2458		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2459		- $per_page = 25;
2460		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2461		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2462		-
2463		- // Get Pinecone manager and options
2464		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
2465		- if (!$pinecone_manager) {
2466		- wp_send_json_error(array('message' => 'Pinecone manager not available'));
2467		- return;
2468		- }
2469		-
2470		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
2471		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2472		- $pinecone_api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2473		-
2474		- if (!$use_pinecone || empty($pinecone_api_key)) {
2475		- wp_send_json_error(array('message' => 'Pinecone not configured'));
2476		- return;
2477		- }
2478		-
2479		- // Fetch records from Pinecone
2480		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, $search_query, $page, $per_page, $bot_id, $content_type_filter);
2481		- $prompts = $records['data'] ?? array();
2482		- $total_records = $records['total'] ?? 0;
2483		-
2484		- // Preprocess Pinecone records — set chunk_metadata and display_content
2485		- // (matches admin-knowledge-page.php preprocessing)
2486		- foreach ($prompts as $prompt) {
2487		- if (isset($prompt->chunk_index) && $prompt->chunk_index !== null) {
2488		- $prompt->chunk_metadata = array(
2489		- 'chunk_index' => intval($prompt->chunk_index),
2490		- 'total_chunks' => isset($prompt->total_chunks) ? intval($prompt->total_chunks) : null,
2491		- 'is_chunked' => isset($prompt->is_chunked) ? (bool) $prompt->is_chunked : true,
2492		- 'source_url' => $prompt->source_url ?? ''
2493		- );
2494		- $prompt->display_content = $prompt->article_content;
2495		- } else {
2496		- $prompt->chunk_metadata = array();
2497		- $prompt->display_content = $prompt->article_content ?? '';
2498		- }
2499		- }
2500		-
2501		- // Group prompts by source_url
2502		- $grouped_prompts = array();
2503		- foreach ($prompts as $prompt) {
2504		- $source_url = '';
2505		- if (!empty($prompt->chunk_metadata['source_url'])) {
2506		- $source_url = $prompt->chunk_metadata['source_url'];
2507		- } elseif (!empty($prompt->source_url)) {
2508		- $source_url = $prompt->source_url;
2509		- }
2510		-
2511		- if (!empty($source_url)) {
2512		- if (!isset($grouped_prompts[$source_url])) {
2513		- $grouped_prompts[$source_url] = array();
2514		- }
2515		- $grouped_prompts[$source_url][] = $prompt;
2516		- } else {
2517		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2518		- }
2519		- }
2520		-
2521		- // Sort each group by chunk_index
2522		- foreach ($grouped_prompts as $source_url => &$group) {
2523		- usort($group, function($a, $b) {
2524		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2525		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2526		- return $index_a - $index_b;
2527		- });
2528		- }
2529		- unset($group);
2530		-
2531		- // Build HTML for the table rows - must match admin-knowledge-page.php structure exactly
2532		- ob_start();
2533		- $display_index = 0;
2534		- $current_page = $page;
2535		- $data_source = 'pinecone';
2536		- $current_bot_id = $bot_id;
2537		- $preview_length = 150;
2538		-
2539		- if (empty($grouped_prompts)) {
2540		- echo '<tr><td colspan="4" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2541		- esc_html_e('No knowledge entries found in Pinecone.', 'mxchat');
2542		- echo '</td></tr>';
2543		- } else {
2544		- foreach ($grouped_prompts as $source_url => $group) {
2545		- $chunk_count = count($group);
2546		- $first_prompt = $group[0];
2547		- $display_index++;
2548		-
2549		- if ($chunk_count > 1) {
2550		- // Multiple chunks - show grouped row with expand button
2551		- $group_id = 'group-' . md5($source_url);
2552		- ?>
2553		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2554		- class="mxchat-chunk-group-header"
2555		- data-source="<?php echo esc_attr($data_source); ?>"
2556		- data-group-id="<?php echo esc_attr($group_id); ?>"
2557		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2558		- <td style="padding: 12px 16px; text-align: center;">
2559		- <input type="checkbox"
2560		- class="mxchat-entry-checkbox"
2561		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2562		- data-source="<?php echo esc_attr($data_source); ?>"
2563		- data-source-url="<?php echo esc_attr($source_url); ?>"
2564		- data-is-group="true"
2565		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2566		- </td>
2567		- <td style="padding: 12px 16px; font-size: 13px;">
2568		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2569		- </td>
2570		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2571		- <div class="mxchat-chunk-group-info">
2572		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2573		- <span class="dashicons dashicons-arrow-right-alt2"></span>
2574		- </button>
2575		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2576		- <span class="mxchat-chunk-preview">
2577		- <?php
2578		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : (isset($first_prompt->article_content) ? $first_prompt->article_content : '');
2579		- $content_preview = mb_substr($parent_content, 0, 100);
2580		- echo esc_html($content_preview . '...');
2581		- ?>
2582		- </span>
2583		- </div>
2584		- </td>
2585		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2586		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2587		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2588		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2589		- <?php esc_html_e('View Source', 'mxchat'); ?>
2590		- </a>
2591		- <?php else : ?>
2592		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2593		- <?php endif; ?>
2594		- </td>
2595		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2596		- <?php if ($data_source !== 'pinecone') : ?>
2597		- <button type="button"
2598		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2599		- data-source-url="<?php echo esc_attr($source_url); ?>"
2600		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2601		- data-data-source="<?php echo esc_attr($data_source); ?>"
2602		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2603		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2604		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2605		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2606		- </button>
2607		- <?php endif; ?>
2608		- <button type="button"
2609		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2610		- data-source-url="<?php echo esc_attr($source_url); ?>"
2611		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2612		- data-data-source="<?php echo esc_attr($data_source); ?>"
2613		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2614		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2615		- style="color: var(--mxch-error);"
2616		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2617		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2618		- </button>
2619		- </td>
2620		- </tr>
2621		- <?php
2622		- // Render hidden chunk rows
2623		- foreach ($group as $chunk_index => $chunk) {
2624		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
2625		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
2626		- $content = isset($chunk->display_content) ? $chunk->display_content : (isset($chunk->article_content) ? $chunk->article_content : '');
2627		- $content_preview = mb_strlen($content) > $preview_length
2628		- ? mb_substr($content, 0, $preview_length) . '...'
2629		- : $content;
2630		- ?>
2631		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
2632		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
2633		- data-source="<?php echo esc_attr($data_source); ?>"
2634		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
2635		- <td style="padding: 12px 16px; text-align: center;">
2636		- <!-- Checkbox column placeholder for chunks (managed by group) -->
2637		- </td>
2638		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
2639		- <!-- Hidden ID column for chunks -->
2640		- </td>
2641		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2642		- <div class="mxchat-accordion-wrapper">
2643		- <div class="mxchat-content-preview">
2644		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
2645		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
2646		- </span>
2647		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2648		- <?php if (mb_strlen($content) > $preview_length) : ?>
2649		- <button class="mxchat-expand-toggle" type="button">
2650		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2651		- </button>
2652		- <?php endif; ?>
2653		- </div>
2654		- <div class="mxchat-content-full" style="display: none;">
2655		- <div class="content-view">
2656		- <?php
2657		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2658		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2659		- echo wp_kses_post(wpautop($content));
2660		- echo '</div>';
2661		- } else {
2662		- echo wp_kses_post(wpautop($content));
2663		- }
2664		- ?>
2665		- </div>
2666		- </div>
2667		- </div>
2668		- </td>
2669		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2670		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
2671		- </td>
2672		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
2673		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
2674		- </td>
2675		- </tr>
2676		- <?php
2677		- }
2678		- } else {
2679		- // Single entry - display normally with accordion
2680		- $prompt = $first_prompt;
2681		- $content = isset($prompt->display_content) ? $prompt->display_content : (isset($prompt->article_content) ? $prompt->article_content : '');
2682		- $content_preview = mb_strlen($content) > $preview_length
2683		- ? mb_substr($content, 0, $preview_length) . '...'
2684		- : $content;
2685		- ?>
2686		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
2687		- data-source="<?php echo esc_attr($data_source); ?>"
2688		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2689		- <td style="padding: 12px 16px; text-align: center;">
2690		- <input type="checkbox"
2691		- class="mxchat-entry-checkbox"
2692		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2693		- data-source="<?php echo esc_attr($data_source); ?>"
2694		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
2695		- data-is-group="false"
2696		- data-chunk-count="1">
2697		- </td>
2698		- <td style="padding: 12px 16px; font-size: 13px;">
2699		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2700		- </td>
2701		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2702		- <div class="mxchat-accordion-wrapper">
2703		- <div class="mxchat-content-preview">
2704		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2705		- <?php if (mb_strlen($content) > $preview_length) : ?>
2706		- <button class="mxchat-expand-toggle" type="button">
2707		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2708		- </button>
2709		- <?php endif; ?>
2710		- </div>
2711		- <div class="mxchat-content-full" style="display: none;">
2712		- <div class="content-view">
2713		- <?php
2714		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2715		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2716		- echo wp_kses_post(wpautop($content));
2717		- echo '</div>';
2718		- } else {
2719		- echo wp_kses_post(wpautop($content));
2720		- }
2721		- ?>
2722		- </div>
2723		- </div>
2724		- </div>
2725		- </td>
2726		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2727		- <?php
2728		- $actual_source = $source_url;
2729		- if (strpos($source_url, '_ungrouped_') === 0) {
2730		- $actual_source = $prompt->source_url ?? '';
2731		- }
2732		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
2733		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2734		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2735		- <?php esc_html_e('View', 'mxchat'); ?>
2736		- </a>
2737		- <?php else : ?>
2738		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
2739		- <?php endif; ?>
2740		- </td>
2741		- <td style="padding: 12px 16px;">
2742		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-ajax" data-vector-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_pinecone_prompt_nonce'); ?>" style="color: var(--mxch-error);">
2743		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2744		- </button>
2745		- </td>
2746		- </tr>
2747		- <?php
2748		- }
2749		- }
2750		- }
2751		- $html = ob_get_clean();
2752		-
2753		- // Generate pagination HTML for Pinecone
2754		- $total_pages = ceil($total_records / $per_page);
2755		- $pagination_html = '';
2756		- if ($total_pages > 1) {
2757		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
2758		-
2759		- // Previous button
2760		- if ($page > 1) {
2761		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
2762		- }
2763		-
2764		- // Page numbers
2765		- $start_page = max(1, $page - 2);
2766		- $end_page = min($total_pages, $page + 2);
2767		-
2768		- if ($start_page > 1) {
2769		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
2770		- if ($start_page > 2) {
2771		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2772		- }
2773		- }
2774		-
2775		- for ($i = $start_page; $i <= $end_page; $i++) {
2776		- if ($i == $page) {
2777		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
2778		- } else {
2779		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
2780		- }
2781		- }
2782		-
2783		- if ($end_page < $total_pages) {
2784		- if ($end_page < $total_pages - 1) {
2785		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2786		- }
2787		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
2788		- }
2789		-
2790		- // Next button
2791		- if ($page < $total_pages) {
2792		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
2793		- }
2794		-
2795		- $pagination_html .= '</div>';
2796		- }
2797		-
2798		- wp_send_json_success(array(
2799		- 'html' => $html,
2800		- 'pagination_html' => $pagination_html,
2801		- 'total_count' => $total_records,
2802		- 'total_pages' => $total_pages,
2803		- 'page' => $page,
2804		- 'per_page' => $per_page,
2805		- 'data_source' => 'pinecone'
2806		- ));
2807		-}
2808		-
2809		-/**
2810		- * AJAX handler for pagination - handles both WordPress DB and Pinecone data sources
2811		- * Returns paginated entries without requiring a full page reload
2812		- */
2813		-public function ajax_mxchat_paginate_entries() {
2814		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2815		-
2816		- if (!current_user_can('manage_options')) {
2817		- wp_send_json_error(array('message' => 'Unauthorized'));
2818		- return;
2819		- }
2820		-
2821		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2822		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2823		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2824		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2825		- $per_page = 25;
2826		-
2827		- // Check if Pinecone is enabled for this bot
2828		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
2829		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
2830		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2831		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
2832		-
2833		- if ($use_pinecone && $has_pinecone_api) {
2834		- // Delegate to Pinecone pagination handler (pass search params)
2835		- $_POST['page'] = $page;
2836		- $_POST['search'] = $search_query;
2837		- $_POST['content_type'] = $content_type_filter;
2838		- $this->ajax_mxchat_refresh_pinecone_entries();
2839		- return;
2840		- }
2841		-
2842		- // WordPress DB pagination - MUST match initial page load logic exactly
2843		- global $wpdb;
2844		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2845		- $offset = ($page - 1) * $per_page;
2846		-
2847		- // Build WHERE clause for search and content type filtering
2848		- $where_clauses = array();
2849		- $where_values = array();
2850		-
2851		- if ($search_query) {
2852		- $where_clauses[] = "article_content LIKE %s";
2853		- $where_values[] = '%' . $wpdb->esc_like($search_query) . '%';
2854		- }
2855		-
2856		- if ($content_type_filter) {
2857		- switch ($content_type_filter) {
2858		- case 'manual':
2859		- $where_clauses[] = "(source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')";
2860		- break;
2861		- case 'pdf':
2862		- $where_clauses[] = "source_url LIKE '%.pdf'";
2863		- break;
2864		- case 'url':
2865		- $where_clauses[] = "source_url != '' AND source_url IS NOT NULL AND source_url NOT LIKE 'mxchat://%' AND source_url NOT LIKE '%.pdf'";
2866		- break;
2867		- }
2868		- }
2869		-
2870		- $where_sql = !empty($where_clauses) ? 'WHERE ' . implode(' AND ', $where_clauses) : '';
2871		-
2872		- // Count grouped entries with filters applied
2873		- if (!empty($where_values)) {
2874		- $count_args = array_merge($where_values, $where_values);
2875		- $count_query = $wpdb->prepare(
2876		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2877		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))",
2878		- ...$count_args
2879		- );
2880		- $total_records = $wpdb->get_var($count_query);
2881		- } else if (!empty($where_sql)) {
2882		- // Content type filter only (no search), no prepared values needed
2883		- $total_records = $wpdb->get_var(
2884		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2885		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))"
2886		- );
2887		- } else {
2888		- // No filters
2889		- $total_records = $wpdb->get_var(
2890		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2891		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
2892		- );
2893		- }
2894		- $total_pages = ceil($total_records / $per_page);
2895		-
2896		- // Step 1: Get unique source_urls for this page (ordered by latest timestamp) with filters
2897		- if (!empty($where_values)) {
2898		- $query_args = array_merge($where_values, array($per_page, $offset));
2899		- $urls_query = $wpdb->prepare(
2900		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2901		- {$where_sql}
2902		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2903		- ...$query_args
2904		- );
2905		- } else if (!empty($where_sql)) {
2906		- $urls_query = $wpdb->prepare(
2907		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2908		- {$where_sql}
2909		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2910		- $per_page, $offset
2911		- );
2912		- } else {
2913		- $urls_query = $wpdb->prepare(
2914		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2915		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2916		- $per_page, $offset
2917		- );
2918		- }
2919		- $page_urls = $wpdb->get_results($urls_query);
2920		-
2921		- // Step 2: Build list of source_urls to fetch
2922		- $url_list = array();
2923		- $url_order_map = array();
2924		- $order_index = 0;
2925		- foreach ($page_urls as $url_row) {
2926		- $url = $url_row->source_url;
2927		- $url_list[] = $url;
2928		- $url_order_map[$url] = $order_index++;
2929		- }
2930		-
2931		- // Step 3: Fetch all rows for these source_urls (with search filter if applicable)
2932		- $prompts = array();
2933		- if (!empty($url_list)) {
2934		- $placeholders = implode(',', array_fill(0, count($url_list), '%s'));
2935		- if ($search_query) {
2936		- // Include search filter in the final fetch
2937		- $prompts_query = $wpdb->prepare(
2938		- "SELECT id, article_content, source_url, timestamp, role_restriction
2939		- FROM {$table_name}
2940		- WHERE source_url IN ($placeholders) AND article_content LIKE %s
2941		- ORDER BY timestamp DESC",
2942		- ...array_merge($url_list, ['%' . $wpdb->esc_like($search_query) . '%'])
2943		- );
2944		- } else {
2945		- $prompts_query = $wpdb->prepare(
2946		- "SELECT id, article_content, source_url, timestamp, role_restriction
2947		- FROM {$table_name}
2948		- WHERE source_url IN ($placeholders)
2949		- ORDER BY timestamp DESC",
2950		- $url_list
2951		- );
2952		- }
2953		- $prompts = $wpdb->get_results($prompts_query);
2954		- }
2955		-
2956		- // Group prompts by source_url for chunk display
2957		- $grouped_prompts = array();
2958		- foreach ($prompts as $prompt) {
2959		- $source_url = $prompt->source_url ?? '';
2960		-
2961		- // Parse chunk metadata using the proper chunker method (same as initial page load)
2962		- if (class_exists('MxChat_Chunker')) {
2963		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($prompt->article_content);
2964		- $prompt->chunk_metadata = $chunk_meta['metadata'];
2965		- $prompt->display_content = $chunk_meta['text'];
2966		- } else {
2967		- $prompt->chunk_metadata = array();
2968		- $prompt->display_content = $prompt->article_content;
2969		- }
2970		-
2971		- if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0) {
2972		- if (!isset($grouped_prompts[$source_url])) {
2973		- $grouped_prompts[$source_url] = array();
2974		- }
2975		- $grouped_prompts[$source_url][] = $prompt;
2976		- } else {
2977		- // Ungrouped entries
2978		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2979		- }
2980		- }
2981		-
2982		- // Sort groups by the original URL order (newest first)
2983		- uksort($grouped_prompts, function($a, $b) use ($url_order_map) {
2984		- $order_a = isset($url_order_map[$a]) ? $url_order_map[$a] : PHP_INT_MAX;
2985		- $order_b = isset($url_order_map[$b]) ? $url_order_map[$b] : PHP_INT_MAX;
2986		- return $order_a - $order_b;
2987		- });
2988		-
2989		- // Sort each group internally by chunk_index
2990		- foreach ($grouped_prompts as $source_url => &$group) {
2991		- usort($group, function($a, $b) {
2992		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2993		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2994		- return $index_a - $index_b;
2995		- });
2996		- }
2997		- unset($group);
2998		-
2999		- // Build HTML for the table rows
3000		- ob_start();
3001		- $display_index = 0;
3002		- $current_page = $page;
3003		- $data_source = 'wordpress';
3004		- $current_bot_id = $bot_id;
3005		- $preview_length = 150;
3006		-
3007		- if (empty($grouped_prompts)) {
3008		- echo '<tr><td colspan="5" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
3009		- esc_html_e('No knowledge entries found. Use the Import Options to add content.', 'mxchat');
3010		- echo '</td></tr>';
3011		- } else {
3012		- foreach ($grouped_prompts as $source_url => $group) {
3013		- $chunk_count = count($group);
3014		- $first_prompt = $group[0];
3015		- $display_index++;
3016		-
3017		- if ($chunk_count > 1) {
3018		- // Multiple chunks - show grouped row with expand button
3019		- $group_id = 'group-' . md5($source_url);
3020		- ?>
3021		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
3022		- class="mxchat-chunk-group-header"
3023		- data-source="<?php echo esc_attr($data_source); ?>"
3024		- data-group-id="<?php echo esc_attr($group_id); ?>"
3025		- style="border-bottom: 1px solid var(--mxch-card-border);">
3026		- <td style="padding: 12px 16px; text-align: center;">
3027		- <input type="checkbox"
3028		- class="mxchat-entry-checkbox"
3029		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
3030		- data-source="<?php echo esc_attr($data_source); ?>"
3031		- data-source-url="<?php echo esc_attr($source_url); ?>"
3032		- data-is-group="true"
3033		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
3034		- </td>
3035		- <td style="padding: 12px 16px; font-size: 13px;">
3036		- <?php echo esc_html($first_prompt->id); ?>
3037		- </td>
3038		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3039		- <div class="mxchat-chunk-group-info">
3040		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
3041		- <span class="dashicons dashicons-arrow-right-alt2"></span>
3042		- </button>
3043		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
3044		- <span class="mxchat-chunk-preview">
3045		- <?php
3046		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : $first_prompt->article_content;
3047		- $content_preview = mb_substr($parent_content, 0, 100);
3048		- echo esc_html($content_preview . '...');
3049		- ?>
3050		- </span>
3051		- </div>
3052		- </td>
3053		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3054		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
3055		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
3056		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
3057		- <?php esc_html_e('View Source', 'mxchat'); ?>
3058		- </a>
3059		- <?php else : ?>
3060		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
3061		- <?php endif; ?>
3062		- </td>
3063		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
3064		- <?php if ($data_source !== 'pinecone') : ?>
3065		- <button type="button"
3066		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
3067		- data-source-url="<?php echo esc_attr($source_url); ?>"
3068		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
3069		- data-data-source="<?php echo esc_attr($data_source); ?>"
3070		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3071		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
3072		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
3073		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
3074		- </button>
3075		- <?php endif; ?>
3076		- <button type="button"
3077		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
3078		- data-source-url="<?php echo esc_attr($source_url); ?>"
3079		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
3080		- data-data-source="<?php echo esc_attr($data_source); ?>"
3081		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3082		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
3083		- style="color: var(--mxch-error);"
3084		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
3085		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
3086		- </button>
3087		- </td>
3088		- </tr>
3089		- <?php
3090		- // Render hidden chunk rows
3091		- foreach ($group as $chunk_index => $chunk) {
3092		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
3093		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
3094		- $content = isset($chunk->display_content) ? $chunk->display_content : $chunk->article_content;
3095		- $content_preview = mb_strlen($content) > $preview_length
3096		- ? mb_substr($content, 0, $preview_length) . '...'
3097		- : $content;
3098		- ?>
3099		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
3100		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
3101		- data-source="<?php echo esc_attr($data_source); ?>"
3102		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
3103		- <td style="padding: 12px 16px; text-align: center;">
3104		- <!-- Checkbox column placeholder for chunks (managed by group) -->
3105		- </td>
3106		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
3107		- <!-- Hidden ID column for chunks -->
3108		- </td>
3109		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3110		- <div class="mxchat-accordion-wrapper">
3111		- <div class="mxchat-content-preview">
3112		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
3113		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
3114		- </span>
3115		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
3116		- <?php if (mb_strlen($content) > $preview_length) : ?>
3117		- <button class="mxchat-expand-toggle" type="button">
3118		- <span class="dashicons dashicons-arrow-down-alt2"></span>
3119		- </button>
3120		- <?php endif; ?>
3121		- </div>
3122		- <div class="mxchat-content-full" style="display: none;">
3123		- <div class="content-view">
3124		- <?php
3125		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
3126		- echo '<div dir="rtl" lang="he" class="rtl-content">';
3127		- echo wp_kses_post(wpautop($content));
3128		- echo '</div>';
3129		- } else {
3130		- echo wp_kses_post(wpautop($content));
3131		- }
3132		- ?>
3133		- </div>
3134		- </div>
3135		- </div>
3136		- </td>
3137		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3138		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
3139		- </td>
3140		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
3141		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
3142		- </td>
3143		- </tr>
3144		- <?php
3145		- }
3146		- } else {
3147		- // Single entry - display normally with accordion
3148		- $prompt = $first_prompt;
3149		- $content = isset($prompt->display_content) ? $prompt->display_content : $prompt->article_content;
3150		- $content_preview = mb_strlen($content) > $preview_length
3151		- ? mb_substr($content, 0, $preview_length) . '...'
3152		- : $content;
3153		- ?>
3154		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
3155		- data-source="<?php echo esc_attr($data_source); ?>"
3156		- style="border-bottom: 1px solid var(--mxch-card-border);">
3157		- <td style="padding: 12px 16px; text-align: center;">
3158		- <input type="checkbox"
3159		- class="mxchat-entry-checkbox"
3160		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
3161		- data-source="<?php echo esc_attr($data_source); ?>"
3162		- data-source-url="<?php echo esc_attr($source_url); ?>"
3163		- data-is-group="false">
3164		- </td>
3165		- <td style="padding: 12px 16px; font-size: 13px;">
3166		- <?php echo esc_html($prompt->id); ?>
3167		- </td>
3168		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3169		- <div class="mxchat-accordion-wrapper">
3170		- <div class="mxchat-content-preview">
3171		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
3172		- <?php if (mb_strlen($content) > $preview_length) : ?>
3173		- <button class="mxchat-expand-toggle" type="button">
3174		- <span class="dashicons dashicons-arrow-down-alt2"></span>
3175		- </button>
3176		- <?php endif; ?>
3177		- </div>
3178		- <div class="mxchat-content-full" style="display: none;">
3179		- <div class="content-view">
3180		- <?php
3181		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
3182		- echo '<div dir="rtl" lang="he" class="rtl-content">';
3183		- echo wp_kses_post(wpautop($content));
3184		- echo '</div>';
3185		- } else {
3186		- echo wp_kses_post(wpautop($content));
3187		- }
3188		- ?>
3189		- </div>
3190		- </div>
3191		- </div>
3192		- </td>
3193		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3194		- <?php
3195		- $actual_source = $source_url;
3196		- if (strpos($source_url, '_ungrouped_') === 0) {
3197		- $actual_source = $prompt->source_url ?? '';
3198		- }
3199		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
3200		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
3201		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
3202		- <?php esc_html_e('View', 'mxchat'); ?>
3203		- </a>
3204		- <?php else : ?>
3205		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
3206		- <?php endif; ?>
3207		- </td>
3208		- <td style="padding: 12px 16px; white-space: nowrap;">
3209		- <button type="button"
3210		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
3211		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
3212		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
3213		- data-data-source="<?php echo esc_attr($data_source); ?>"
3214		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3215		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
3216		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
3217		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
3218		- </button>
3219		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-wordpress" data-entry-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_wordpress_prompt_nonce'); ?>" style="color: var(--mxch-error);">
3220		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
3221		- </button>
3222		- </td>
3223		- </tr>
3224		- <?php
3225		- }
3226		- }
3227		- }
3228		- $html = ob_get_clean();
3229		-
3230		- // Generate pagination HTML (include search/filter data for subsequent pages)
3231		- $pagination_html = '';
3232		- if ($total_pages > 1) {
3233		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
3234		-
3235		- // Previous button
3236		- if ($page > 1) {
3237		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
3238		- }
3239		-
3240		- // Page numbers
3241		- $start_page = max(1, $page - 2);
3242		- $end_page = min($total_pages, $page + 2);
3243		-
3244		- if ($start_page > 1) {
3245		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
3246		- if ($start_page > 2) {
3247		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
3248		- }
3249		- }
3250		-
3251		- for ($i = $start_page; $i <= $end_page; $i++) {
3252		- if ($i == $page) {
3253		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
3254		- } else {
3255		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
3256		- }
3257		- }
3258		-
3259		- if ($end_page < $total_pages) {
3260		- if ($end_page < $total_pages - 1) {
3261		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
3262		- }
3263		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
3264		- }
3265		-
3266		- // Next button
3267		- if ($page < $total_pages) {
3268		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
3269		- }
3270		-
3271		- $pagination_html .= '</div>';
3272		- }
3273		-
3274		- wp_send_json_success(array(
3275		- 'html' => $html,
3276		- 'pagination_html' => $pagination_html,
3277		- 'total_count' => $total_records,
3278		- 'total_pages' => $total_pages,
3279		- 'page' => $page,
3280		- 'per_page' => $per_page,
3281		- 'data_source' => 'wordpress'
3282		- ));
3283		-}
3284		-
3285		-/**
3286		- * AJAX handler to detect available sitemaps on the site
3287		- * Optimized for speed - only checks primary sitemap indexes first
3288		- */
3289		-public function ajax_mxchat_detect_sitemaps() {
3290		- check_ajax_referer('mxchat_detect_sitemaps_nonce', 'nonce');
3291		-
3292		- if (!current_user_can('manage_options')) {
3293		- wp_send_json_error(array('message' => 'Unauthorized'));
3294		- return;
3295		- }
3296		-
3297		- $site_url = get_site_url();
3298		- $sitemaps = array();
3299		- $found_index = false;
3300		-
3301		- // Only check the main sitemap index files first (much faster)
3302		- // These are the primary entry points that contain sub-sitemaps
3303		- $primary_indexes = array(
3304		- 'sitemap_index.xml' => 'Yoast SEO / Rank Math', // Yoast & Rank Math
3305		- 'wp-sitemap.xml' => 'WordPress Core', // WordPress Core
3306		- 'sitemap.xml' => 'Standard', // Generic/AIOSEO
3307		- );
3308		-
3309		- foreach ($primary_indexes as $path => $source) {
3310		- $url = trailingslashit($site_url) . $path;
3311		-
3312		- $response = wp_remote_head($url, array(
3313		- 'timeout' => 10,
3314		- 'sslverify' => false,
3315		- 'redirection' => 1,
3316		- 'user-agent' => mxchat_ingest_user_agent(),
3317		- ));
3318		-
3319		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3320		- // Found a sitemap index - parse it to get sub-sitemaps
3321		- $sub_sitemaps = $this->parse_sitemap_index($url);
3322		- if (!empty($sub_sitemaps)) {
3323		- $sitemaps[] = array(
3324		- 'url' => $url,
3325		- 'type' => 'index',
3326		- 'source' => $source,
3327		- 'sub_sitemaps' => $sub_sitemaps
3328		- );
3329		- $found_index = true;
3330		- // Found a valid index, no need to check others
3331		- break;
3332		- }
3333		- }
3334		- }
3335		-
3336		- // If no sitemap index found, check for standalone sitemaps
3337		- if (!$found_index) {
3338		- $standalone_sitemaps = array(
3339		- 'post-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
3340		- 'page-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
3341		- );
3342		-
3343		- foreach ($standalone_sitemaps as $path => $info) {
3344		- $url = trailingslashit($site_url) . $path;
3345		-
3346		- $response = wp_remote_head($url, array(
3347		- 'timeout' => 2,
3348		- 'sslverify' => false
3349		- ));
3350		-
3351		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3352		- $sitemaps[] = array(
3353		- 'url' => $url,
3354		- 'type' => $info['type'],
3355		- 'source' => $info['source'],
3356		- 'url_count' => 0 // Skip URL count for speed
3357		- );
3358		- }
3359		- }
3360		- }
3361		-
3362		- wp_send_json_success(array(
3363		- 'sitemaps' => $sitemaps,
3364		- 'site_url' => $site_url
3365		- ));
3366		-}
3367		-
3368		-/**
3369		- * Parse a sitemap index to get sub-sitemaps
3370		- * Optimized: doesn't fetch URL count for each sub-sitemap (too slow)
3371		- */
3372		-private function parse_sitemap_index($url) {
3373		- $sub_sitemaps = array();
3374		-
3375		- $response = wp_remote_get($url, array(
3376		- 'timeout' => 30,
3377		- 'sslverify' => false,
3378		- 'user-agent' => mxchat_ingest_user_agent(),
3379		- 'headers' => array(
3380		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
3381		- ),
3382		- ));
3383		-
3384		- if (is_wp_error($response)) {
3385		- return $sub_sitemaps;
3386		- }
3387		-
3388		- $body = wp_remote_retrieve_body($response);
3389		- if (empty($body)) {
3390		- return $sub_sitemaps;
3391		- }
3392		-
3393		- // Suppress XML errors
3394		- libxml_use_internal_errors(true);
3395		- $xml = simplexml_load_string($body);
3396		- libxml_clear_errors();
3397		-
3398		- if ($xml === false) {
3399		- return $sub_sitemaps;
3400		- }
3401		-
3402		- // Check if it's a sitemap index (contains <sitemap> elements)
3403		- if (isset($xml->sitemap)) {
3404		- foreach ($xml->sitemap as $sitemap) {
3405		- $loc = (string) $sitemap->loc;
3406		- if (!empty($loc)) {
3407		- // Try to determine the type from the URL
3408		- $type = 'content';
3409		- if (strpos($loc, 'category') !== false || strpos($loc, 'tag') !== false || strpos($loc, 'taxonomy') !== false) {
3410		- $type = 'taxonomy';
3411		- } elseif (strpos($loc, 'author') !== false || strpos($loc, 'user') !== false) {
3412		- $type = 'author';
3413		- }
3414		-
3415		- // Skip URL count - too slow to fetch for each sitemap
3416		- $sub_sitemaps[] = array(
3417		- 'url' => $loc,
3418		- 'type' => $type,
3419		- 'url_count' => 0, // Don't fetch - takes too long
3420		- 'name' => basename(parse_url($loc, PHP_URL_PATH))
3421		- );
3422		- }
3423		- }
3424		- }
3425		-
3426		- return $sub_sitemaps;
3427		-}
3428		-
3429		-/**
3430		- * Get URL count from a sitemap
3431		- */
3432		-private function get_sitemap_url_count($url) {
3433		- $response = wp_remote_get($url, array(
3434		- 'timeout' => 30,
3435		- 'sslverify' => false,
3436		- 'user-agent' => mxchat_ingest_user_agent(),
3437		- 'headers' => array(
3438		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
3439		- ),
3440		- ));
3441		-
3442		- if (is_wp_error($response)) {
3443		- return 0;
3444		- }
3445		-
3446		- $body = wp_remote_retrieve_body($response);
3447		- if (empty($body)) {
3448		- return 0;
3449		- }
3450		-
3451		- // Count <url> or <loc> elements
3452		- $count = preg_match_all('/<url>/i', $body, $matches);
3453		- return $count ?: 0;
3454		-}
3455		-
3456		-/**
3457		- * Get sitemaps declared in robots.txt
3458		- */
3459		-private function get_sitemaps_from_robots($site_url) {
3460		- $sitemaps = array();
3461		- $robots_url = trailingslashit($site_url) . 'robots.txt';
3462		-
3463		- $response = wp_remote_get($robots_url, array(
3464		- 'timeout' => 15,
3465		- 'sslverify' => false,
3466		- 'user-agent' => mxchat_ingest_user_agent(),
3467		- ));
3468		-
3469		- if (is_wp_error($response)) {
3470		- return $sitemaps;
3471		- }
3472		-
3473		- $body = wp_remote_retrieve_body($response);
3474		- if (empty($body)) {
3475		- return $sitemaps;
3476		- }
3477		-
3478		- // Find Sitemap: declarations
3479		- if (preg_match_all('/^Sitemap:\s*(.+)$/mi', $body, $matches)) {
3480		- foreach ($matches[1] as $sitemap_url) {
3481		- $sitemap_url = trim($sitemap_url);
3482		- if (filter_var($sitemap_url, FILTER_VALIDATE_URL)) {
3483		- $sitemaps[] = $sitemap_url;
3484		- }
3485		- }
3486		- }
3487		-
3488		- return $sitemaps;
3489		-}
3490		-
3491	1777	public function mxchat_stop_processing() {
3492	1778	// Verify permissions
3493	1779	if (!current_user_can('manage_options')) {
3494	1780	wp_die(esc_html__('Unauthorized access', 'mxchat'));
		@@ -3496,52 +1782,28 @@
3496	1782
3497	1783	// Verify nonce
3498	1784	check_admin_referer('mxchat_stop_processing_action', 'mxchat_stop_processing_nonce');
3499	1785
3500		- global $wpdb;
3501		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
3502		-
3503		- // Get active queue IDs
3504		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
3505		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
3506		-
3507		- // Delete all pending items from active queues
3508		- if ($sitemap_queue_id) {
3509		- $wpdb->delete(
3510		- $table_name,
3511		- array(
3512		- 'queue_id' => $sitemap_queue_id,
3513		- 'status' => 'pending'
3514		- ),
3515		- array('%s', '%s')
3516		- );
3517		-
3518		- delete_transient('mxchat_active_queue_sitemap');
	1786	+ // Get the last sitemap URL and clear its transient
	1787	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
	1788	+ if ($sitemap_url) {
	1789	+ delete_transient('mxchat_sitemap_status_' . md5($sitemap_url));
3519	1790	delete_transient('mxchat_last_sitemap_url');
3520	1791	}
3521		-
3522		- if ($pdf_queue_id) {
3523		- // Get PDF path before deleting
3524		- $pdf_path = $this->mxchat_get_queue_meta($pdf_queue_id, 'pdf_path');
3525		-
3526		- $wpdb->delete(
3527		- $table_name,
3528		- array(
3529		- 'queue_id' => $pdf_queue_id,
3530		- 'status' => 'pending'
3531		- ),
3532		- array('%s', '%s')
3533		- );
3534		-
3535		- // Delete PDF file
3536		- if ($pdf_path && file_exists($pdf_path)) {
3537		- wp_delete_file($pdf_path);
3538		- }
3539		-
3540		- delete_transient('mxchat_active_queue_pdf');
	1792	+
	1793	+ // Get the last PDF URL and clear its transient
	1794	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1795	+ if ($pdf_url) {
	1796	+ delete_transient('mxchat_pdf_status_' . md5($pdf_url));
3541	1797	delete_transient('mxchat_last_pdf_url');
3542	1798	}
3543	1799
	1800	+ // Unschedule any pending sitemap events
	1801	+ $timestamp = wp_next_scheduled('mxchat_process_sitemap_urls');
	1802	+ if ($timestamp) {
	1803	+ wp_unschedule_event($timestamp, 'mxchat_process_sitemap_urls');
	1804	+ }
	1805	+
3544	1806	// Redirect back with a success message
3545	1807	set_transient('mxchat_admin_notice_success',
3546	1808	esc_html__('Processing has been stopped successfully.', 'mxchat'),
3547	1809	30
		@@ -3548,12 +1810,8 @@
3548	1810	);
3549	1811	wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
3550	1812	exit;
3551	1813	}
3552		-
3553		-/**
3554		- * Get content list for processing
3555		- */
3556	1814	public function ajax_mxchat_get_content_list() {
3557	1815	// Verify the nonce
3558	1816	check_ajax_referer('mxchat_content_selector_nonce', 'nonce');
3559	1817
		@@ -3561,9 +1819,9 @@
3561	1819	wp_send_json_error(__('Unauthorized access', 'mxchat'));
3562	1820	}
3563	1821
3564	1822	$page = isset($_GET['page']) ? absint($_GET['page']) : 1;
3565		- $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 100;
	1823	+ $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 20;
3566	1824	$search = isset($_GET['search']) ? sanitize_text_field($_GET['search']) : '';
3567	1825	$post_type = isset($_GET['post_type']) ? sanitize_text_field($_GET['post_type']) : 'all';
3568	1826	$post_status = isset($_GET['post_status']) ? sanitize_text_field($_GET['post_status']) : 'publish';
3569	1827	$processed_filter = isset($_GET['processed_filter']) ? sanitize_text_field($_GET['processed_filter']) : 'all';
		@@ -3576,47 +1834,20 @@
3576	1834	'orderby' => 'date',
3577	1835	'order' => 'DESC',
3578	1836	);
3579	1837
3580		- // Handle post types - IMPROVED VERSION
	1838	+ // Handle post types
3581	1839	if ($post_type !== 'all') {
3582	1840	$args['post_type'] = $post_type;
3583	1841	} else {
3584		- // Get all available post types that might contain content
3585		- $all_post_types = array();
	1842	+ // Default to post and page if we can't get post types
	1843	+ $args['post_type'] = array('post', 'page');
3586	1844
3587		- // First get all public post types
3588		- $public_types = get_post_types(array('public' => true), 'names');
3589		- $all_post_types = array_merge($all_post_types, $public_types);
3590		-
3591		- // Add common forum/community post types
3592		- $forum_types = array('topic', 'reply', 'forum', 'wpforo_topic', 'wpforo_post');
3593		- foreach ($forum_types as $forum_type) {
3594		- if (post_type_exists($forum_type)) {
3595		- $all_post_types[] = $forum_type;
3596		- }
	1845	+ // Try to get public post types
	1846	+ $public_types = $this->mxchat_get_public_post_types();
	1847	+ if (is_array($public_types) && !empty($public_types)) {
	1848	+ $args['post_type'] = array_keys($public_types);
3597	1849	}
3598		-
3599		- // Add other commonly used post types
3600		- $common_types = array('product', 'job_listing', 'event', 'portfolio');
3601		- foreach ($common_types as $common_type) {
3602		- if (post_type_exists($common_type)) {
3603		- $all_post_types[] = $common_type;
3604		- }
3605		- }
3606		-
3607		- // Remove duplicates and ensure we have at least some post types
3608		- $all_post_types = array_unique($all_post_types);
3609		-
3610		- if (empty($all_post_types)) {
3611		- // Fallback to basic post types
3612		- $all_post_types = array('post', 'page');
3613		- }
3614		-
3615		- $args['post_type'] = $all_post_types;
3616		-
3617		- // Debug logging to see what post types are being queried
3618		- //error_log('MxChat Debug: Querying post types: ' . implode(', ', $all_post_types));
3619	1850	}
3620	1851
3621	1852	if (!empty($search)) {
3622	1853	$args['s'] = $search;
		@@ -3621,58 +1852,37 @@
3621	1852	if (!empty($search)) {
3622	1853	$args['s'] = $search;
3623	1854	}
3624	1855
3625		- // Get processed data from storage
	1856	+ // ================================
	1857	+ // FIXED: Check only the ACTIVE storage method
	1858	+ // ================================
	1859	+
3626	1860	$processed_data = array();
3627	1861
	1862	+ $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
3628	1863	$pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	1864	+ $pinecone_manager->mxchat_refresh_after_new_content($pinecone_options);
3629	1865	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3630	1866
3631	1867	if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3632		- // Get fresh data from Pinecone - no caching
	1868	+ // ONLY check Pinecone if it's enabled
3633	1869	$processed_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
3634	1870	} else {
3635		- // WordPress DB checking with better URL matching for all post types
	1871	+ // ONLY check WordPress DB if Pinecone is not enabled
3636	1872	global $wpdb;
3637	1873	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3638		- $processed_items = $wpdb->get_results("SELECT id, source_url, article_content, timestamp FROM {$table_name}");
3639		-
3640		- // Group items by source_url to count chunks
3641		- $url_chunk_counts = array();
3642		- $url_latest_timestamp = array();
3643		- $url_first_id = array();
3644		-
	1874	+ $processed_items = $wpdb->get_results("SELECT id, source_url, timestamp FROM {$table_name}");
	1875	+
3645	1876	if (!empty($processed_items)) {
3646	1877	foreach ($processed_items as $item) {
3647		- $url = $item->source_url;
3648		- if (empty($url)) continue;
3649		-
3650		- // Count chunks per URL
3651		- if (!isset($url_chunk_counts[$url])) {
3652		- $url_chunk_counts[$url] = 0;
3653		- $url_latest_timestamp[$url] = $item->timestamp;
3654		- $url_first_id[$url] = $item->id;
3655		- }
3656		- $url_chunk_counts[$url]++;
3657		-
3658		- // Track latest timestamp
3659		- if (strtotime($item->timestamp) > strtotime($url_latest_timestamp[$url])) {
3660		- $url_latest_timestamp[$url] = $item->timestamp;
3661		- }
3662		- }
3663		-
3664		- // Now build processed_data with chunk counts
3665		- foreach ($url_chunk_counts as $url => $chunk_count) {
3666		- $post_id = $this->mxchat_url_to_post_id_improved($url);
3667		-
	1878	+ $post_id = url_to_postid($item->source_url);
3668	1879	if ($post_id) {
3669	1880	$processed_data[$post_id] = array(
3670		- 'db_id' => $url_first_id[$url],
3671		- 'timestamp' => $url_latest_timestamp[$url],
3672		- 'url' => $url,
3673		- 'source' => 'wordpress',
3674		- 'chunk_count' => $chunk_count
	1881	+ 'db_id' => $item->id,
	1882	+ 'timestamp' => $item->timestamp,
	1883	+ 'url' => $item->source_url,
	1884	+ 'source' => 'wordpress'
3675	1885	);
3676	1886	}
3677	1887	}
3678	1888	}
		@@ -3677,8 +1887,10 @@
3677	1887	}
3678	1888	}
3679	1889	}
3680	1890
	1891	+ // ================================
	1892	+
3681	1893	// Get processed IDs as a simple array for in_array checks
3682	1894	$processed_ids = array_keys($processed_data);
3683	1895
3684	1896	// Handle processed/unprocessed filter
		@@ -3720,14 +1932,8 @@
3720	1932	$db_record_id = $item_data['db_id'];
3721	1933	}
3722	1934	}
3723	1935
3724		- // Get chunk count for this item
3725		- $chunk_count = 0;
3726		- if ($is_processed && isset($processed_data[$id]['chunk_count'])) {
3727		- $chunk_count = intval($processed_data[$id]['chunk_count']);
3728		- }
3729		-
3730	1936	$content_items[] = array(
3731	1937	'id' => $id,
3732	1938	'title' => get_the_title(),
3733	1939	'permalink' => get_permalink(),
		@@ -3738,10 +1944,9 @@
3738	1944	'word_count' => $word_count,
3739	1945	'already_processed' => $is_processed,
3740	1946	'processed_date' => $processed_date,
3741	1947	'db_record_id' => $db_record_id,
3742		- 'data_source' => $data_source,
3743		- 'chunk_count' => $chunk_count
	1948	+ 'data_source' => $data_source
3744	1949	);
3745	1950	}
3746	1951	wp_reset_postdata();
3747	1952	}
		@@ -3757,178 +1962,8 @@
3757	1962	wp_send_json_success($response);
3758	1963	exit;
3759	1964	}
3760	1965
3761		-
3762		-/**
3763		- * This function handles various WooCommerce URL formats and permalink structures
3764		- */
3765		-private function mxchat_url_to_post_id_improved($url) {
3766		- // First try the standard WordPress function
3767		- $post_id = url_to_postid($url);
3768		-
3769		- if ($post_id > 0) {
3770		- return $post_id;
3771		- }
3772		-
3773		- // If that fails, try more aggressive URL matching
3774		- // Remove trailing slashes and query parameters for better matching
3775		- $clean_url = rtrim($url, '/');
3776		- $clean_url = strtok($clean_url, '?'); // Remove query parameters
3777		-
3778		- // Try again with cleaned URL
3779		- $post_id = url_to_postid($clean_url);
3780		- if ($post_id > 0) {
3781		- return $post_id;
3782		- }
3783		-
3784		- // For bbPress forum topics, try extracting slug from URL
3785		- if (strpos($url, '/topic/') !== false || strpos($url, '/forums/') !== false) {
3786		- // Handle bbPress URLs: /forums/topic/topic-name/
3787		- if (preg_match('/\/forums\/topic\/([^\/\?]+)/', $url, $matches)) {
3788		- $topic_slug = $matches[1];
3789		-
3790		- // Look up topic by slug
3791		- $topic = get_page_by_path($topic_slug, OBJECT, 'topic');
3792		- if ($topic) {
3793		- return $topic->ID;
3794		- }
3795		-
3796		- // Alternative method: query by post_name
3797		- global $wpdb;
3798		- $post_id = $wpdb->get_var($wpdb->prepare(
3799		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3800		- $topic_slug
3801		- ));
3802		-
3803		- if ($post_id) {
3804		- return intval($post_id);
3805		- }
3806		- }
3807		-
3808		- // Handle simpler topic URLs: /topic/topic-name/
3809		- if (preg_match('/\/topic\/([^\/\?]+)/', $url, $matches)) {
3810		- $topic_slug = $matches[1];
3811		-
3812		- global $wpdb;
3813		- $post_id = $wpdb->get_var($wpdb->prepare(
3814		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3815		- $topic_slug
3816		- ));
3817		-
3818		- if ($post_id) {
3819		- return intval($post_id);
3820		- }
3821		- }
3822		- }
3823		-
3824		- // For WooCommerce products
3825		- if (strpos($url, '/product/') !== false || strpos($url, 'product=') !== false) {
3826		- // Extract product slug from various URL formats
3827		- $product_slug = '';
3828		-
3829		- // Handle pretty permalinks: /product/product-name/
3830		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
3831		- $product_slug = $matches[1];
3832		- }
3833		- // Handle query parameters: ?product=product-name
3834		- elseif (preg_match('/[\?&]product=([^&]+)/', $url, $matches)) {
3835		- $product_slug = $matches[1];
3836		- }
3837		-
3838		- if (!empty($product_slug)) {
3839		- // Look up product by slug
3840		- $product = get_page_by_path($product_slug, OBJECT, 'product');
3841		- if ($product) {
3842		- return $product->ID;
3843		- }
3844		-
3845		- // Alternative method: query by post_name
3846		- global $wpdb;
3847		- $post_id = $wpdb->get_var($wpdb->prepare(
3848		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'product' AND post_status = 'publish'",
3849		- $product_slug
3850		- ));
3851		-
3852		- if ($post_id) {
3853		- return intval($post_id);
3854		- }
3855		- }
3856		- }
3857		-
3858		- // Generic approach: try to extract slug and match against all post types
3859		- $parsed_url = wp_parse_url($clean_url);
3860		- $path = $parsed_url['path'] ?? '';
3861		-
3862		- if (!empty($path)) {
3863		- // Get the last part of the path as potential slug
3864		- $path_parts = array_filter(explode('/', trim($path, '/')));
3865		- $potential_slug = end($path_parts);
3866		-
3867		- if (!empty($potential_slug)) {
3868		- global $wpdb;
3869		-
3870		- // Try to find any post with this slug
3871		- $post_id = $wpdb->get_var($wpdb->prepare(
3872		- "SELECT ID FROM {$wpdb->posts}
3873		- WHERE post_name = %s
3874		- AND post_status IN ('publish', 'closed', 'private')
3875		- AND post_type NOT IN ('revision', 'attachment', 'nav_menu_item')
3876		- ORDER BY CASE
3877		- WHEN post_type = 'post' THEN 1
3878		- WHEN post_type = 'page' THEN 2
3879		- WHEN post_type = 'topic' THEN 3
3880		- WHEN post_type = 'product' THEN 4
3881		- ELSE 5
3882		- END
3883		- LIMIT 1",
3884		- $potential_slug
3885		- ));
3886		-
3887		- if ($post_id) {
3888		- return intval($post_id);
3889		- }
3890		- }
3891		- }
3892		-
3893		- // ADDITIONAL: Try direct database lookup by URL variations
3894		- global $wpdb;
3895		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3896		-
3897		- // Try variations of the URL (with/without trailing slash, http/https)
3898		- $url_variations = array(
3899		- $url,
3900		- rtrim($url, '/'),
3901		- $url . '/',
3902		- str_replace('http://', 'https://', $url),
3903		- str_replace('https://', 'http://', $url),
3904		- str_replace('http://', 'https://', rtrim($url, '/')),
3905		- str_replace('https://', 'http://', rtrim($url, '/'))
3906		- );
3907		-
3908		- // Remove duplicates
3909		- $url_variations = array_unique($url_variations);
3910		-
3911		- foreach ($url_variations as $variation) {
3912		- $existing_record = $wpdb->get_row($wpdb->prepare(
3913		- "SELECT id, source_url FROM $table_name WHERE source_url = %s",
3914		- $variation
3915		- ));
3916		-
3917		- if ($existing_record) {
3918		- // Try to get post ID from this stored URL
3919		- $stored_post_id = url_to_postid($existing_record->source_url);
3920		- if ($stored_post_id > 0) {
3921		- return $stored_post_id;
3922		- }
3923		- }
3924		- }
3925		-
3926		- return 0; // No match found
3927		-}
3928		-/**
3929		- * Process selected content via AJAX
3930		- */
3931	1966	public function ajax_mxchat_process_selected_content() {
3932	1967	// Basic request validation
3933	1968	if (!check_ajax_referer('mxchat_content_selector_nonce', 'nonce', false)) {
3934	1969	wp_send_json_error('Invalid nonce');
		@@ -3952,24 +1987,8 @@
3952	1987	wp_send_json_error('No content selected');
3953	1988	exit;
3954	1989	}
3955	1990
3956		- // Get bot_id from request
3957		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
3958		-
3959		- // ACF→PDF extraction is opt-in per import batch. Persist the last-used value so users
3960		- // don't re-check on every batch; the default is OFF for installs that haven't set it.
3961		- $extract_acf_pdfs = !empty($_POST['extract_acf_pdfs']) && $_POST['extract_acf_pdfs'] !== 'false';
3962		- $mxchat_options = get_option('mxchat_options', array());
3963		- if (!is_array($mxchat_options)) {
3964		- $mxchat_options = array();
3965		- }
3966		- $prior_default = !empty($mxchat_options['acf_pdf_extract_default']);
3967		- if ($prior_default !== $extract_acf_pdfs) {
3968		- $mxchat_options['acf_pdf_extract_default'] = $extract_acf_pdfs ? 1 : 0;
3969		- update_option('mxchat_options', $mxchat_options);
3970		- }
3971		-
3972	1991	// Process only ONE post at a time to avoid request size issues
3973	1992	$post_id = reset($post_ids);
3974	1993	$post = get_post($post_id);
3975	1994
		@@ -3976,185 +1995,15 @@
3976	1995	if (!$post) {
3977	1996	wp_send_json_error('Post not found');
3978	1997	exit;
3979	1998	}
3980		-
3981		- // Allow developers to modify post data before processing into knowledge base
3982		- $post = apply_filters('mxchat_before_process_post', $post, $bot_id);
3983		-
3984		- // Get content including title, short description (for WooCommerce), and main content
3985		- $content = $post->post_title . "\n\n";
3986		-
3987		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
3988		- if (!empty($post->post_excerpt)) {
3989		- // Remove shortcode tags but preserve content inside them
3990		- $clean_excerpt = $this->strip_shortcode_tags_preserve_content($post->post_excerpt);
3991		- $content .= "Short Description: " . wp_strip_all_tags($clean_excerpt) . "\n\n";
3992		- }
3993		-
3994		- // Add main content - remove shortcode tags but preserve content inside them
3995		- $clean_content = $this->strip_shortcode_tags_preserve_content($post->post_content);
3996		- $content .= wp_strip_all_tags($clean_content);
3997		-
3998		- // ADD WOOCOMMERCE PRODUCT DATA (pricing, stock, categories, custom tabs)
3999		- if (get_post_type($post_id) === 'product' && class_exists('WooCommerce')) {
4000		- $product = wc_get_product($post_id);
4001		-
4002		- if ($product) {
4003		- // Get pricing information
4004		- $regular_price = $product->get_regular_price();
4005		- $sale_price = $product->get_sale_price();
4006		- $price = $product->get_price();
4007		- $sku = $product->get_sku();
4008		-
4009		- // Get currency symbol
4010		- $currency_symbol = get_woocommerce_currency_symbol();
4011		-
4012		- // Add pricing information
4013		- $content .= "\n";
4014		- if (!empty($regular_price)) {
4015		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
4016		- } elseif (!empty($price)) {
4017		- $content .= "Price: " . $currency_symbol . $price . "\n";
4018		- }
4019		-
4020		- if (!empty($sale_price) && $sale_price !== $regular_price) {
4021		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
4022		- }
4023		-
4024		- // Handle variable products - show price range
4025		- if ($product->is_type('variable')) {
4026		- $min_price = $product->get_variation_price('min');
4027		- $max_price = $product->get_variation_price('max');
4028		- if ($min_price !== $max_price) {
4029		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
4030		- }
4031		- }
4032		-
4033		- if (!empty($sku)) {
4034		- $content .= "SKU: " . $sku . "\n";
4035		- }
4036		-
4037		- // Get product categories
4038		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
4039		- if (!empty($categories) && !is_wp_error($categories)) {
4040		- $content .= "Categories: " . implode(', ', $categories) . "\n";
4041		- }
4042		- }
4043		-
4044		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
4045		- $custom_tabs = get_post_meta($post_id, 'yikes_woo_products_tabs', true);
4046		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
4047		- foreach ($custom_tabs as $tab) {
4048		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
4049		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
4050		-
4051		- if (!empty($tab_title) && !empty($tab_content)) {
4052		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
4053		- }
4054		- }
4055		- }
4056		-
4057		- // Also check for reusable/saved tabs applied to this product
4058		- $applied_saved_tabs = get_post_meta($post_id, 'yikes_woo_reusable_products_tabs_applied', true);
4059		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
4060		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
4061		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
4062		- foreach ($applied_saved_tabs as $saved_tab_id) {
4063		- if (isset($saved_tabs[$saved_tab_id])) {
4064		- $tab = $saved_tabs[$saved_tab_id];
4065		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
4066		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
4067		-
4068		- if (!empty($tab_title) && !empty($tab_content)) {
4069		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
4070		- }
4071		- }
4072		- }
4073		- }
4074		- }
4075		- }
4076		-
4077		- // ADD ACF FIELDS SUPPORT
4078		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
4079		- $pdf_extracted_count = 0;
4080		- if (!empty($acf_fields)) {
4081		- $acf_content_parts = array();
4082		- $pdf_attachment_ids = array();
4083		-
4084		- foreach ($acf_fields as $field_name => $field_value) {
4085		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
4086		-
4087		- if (!empty($formatted_value)) {
4088		- $field_label = ucwords(str_replace('_', ' ', $field_name));
4089		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
4090		- }
4091		-
4092		- // Walk this field's value tree for any PDF attachment references and queue them for extraction.
4093		- // Only when the user opted into ACF→PDF extraction for this batch; otherwise the ACF text
4094		- // still lands in the KB but the heavier PDF parsing is skipped.
4095		- if ($extract_acf_pdfs) {
4096		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
4097		- }
4098		- }
4099		-
4100		- if (!empty($acf_content_parts)) {
4101		- $content .= "\n\n" . implode("\n", $acf_content_parts);
4102		- }
4103		-
4104		- // Extract text from each unique PDF found in ACF fields and append as a labeled section
4105		- if ($extract_acf_pdfs && !empty($pdf_attachment_ids)) {
4106		- $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
4107		- $pdf_sections = array();
4108		- foreach ($pdf_attachment_ids as $att_id) {
4109		- $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
4110		- if (!empty($pdf_text)) {
4111		- $pdf_title = get_the_title($att_id);
4112		- $pdf_url = wp_get_attachment_url($att_id);
4113		- $header = 'PDF Attachment';
4114		- if (!empty($pdf_title)) {
4115		- $header .= ': ' . $pdf_title;
4116		- }
4117		- if (!empty($pdf_url)) {
4118		- $header .= ' (' . $pdf_url . ')';
4119		- }
4120		- $pdf_sections[] = $header . "\n" . $pdf_text;
4121		- $pdf_extracted_count++;
4122		- }
4123		- }
4124		- if (!empty($pdf_sections)) {
4125		- $content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
4126		- }
4127		- }
4128		- }
4129		-
4130		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
4131		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
4132		- if (!empty($custom_meta)) {
4133		- $meta_content_parts = array();
4134		-
4135		- foreach ($custom_meta as $meta_key => $meta_value) {
4136		- // Convert meta key to readable label
4137		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
4138		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
4139		- }
4140		-
4141		- if (!empty($meta_content_parts)) {
4142		- $content .= "\n\n" . implode("\n", $meta_content_parts);
4143		- }
4144		- }
4145		-
4146		- // Debug logging for WordPress Import content
4147		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Post ID: ' . $post_id . ' Title: ' . $post->post_title);
4148		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Raw post_content length: ' . strlen($post->post_content));
4149		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Final content length: ' . strlen($content));
4150		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Content preview: ' . substr($content, 0, 300));
4151		-
4152		- // Note: Removed 10,000 char limit - chunking now handles large content properly
4153		-
4154		- // Get bot-specific API key
4155		- $bot_options = $this->get_bot_options($bot_id);
4156		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	1999	+
	2000	+ // Get minimal content
	2001	+ $content = $post->post_title . "\n\n" . wp_strip_all_tags($post->post_content);
	2002	+ $content = substr($content, 0, 10000); // Limit content size
	2003	+
	2004	+ // Get API key with proper model detection
	2005	+ $options = get_option('mxchat_options');
4157	2006	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4158	2007
4159	2008	if (strpos($selected_model, 'voyage') === 0) {
4160	2009	$api_key = $options['voyage_api_key'] ?? '';
		@@ -4167,9 +2016,8 @@
4167	2016	$provider_name = 'OpenAI';
4168	2017	}
4169	2018
4170	2019	if (empty($api_key)) {
4171		- MxChat_Admin::mxchat_log_debug('api_error', $provider_name . ' API key not configured for knowledge processing');
4172	2020	wp_send_json_error($provider_name . ' API key not configured');
4173	2021	exit;
4174	2022	}
4175	2023
		@@ -4175,23 +2023,26 @@
4175	2023
4176	2024	$source_url = get_permalink($post_id);
4177	2025	$vector_id = md5($source_url); // Vector ID for Pinecone
4178	2026
4179		- // Check for existing content in bot-specific storage
	2027	+ // ================================
	2028	+ // FIXED: Check for existing content in ONLY the active storage method
	2029	+ // ================================
	2030	+
4180	2031	$is_update = false;
4181	2032
4182		- // Get bot-specific Pinecone configuration
4183		- $bot_pinecone_config = $this->get_bot_pinecone_config($bot_id);
4184		- $use_pinecone = !empty($bot_pinecone_config) && ($bot_pinecone_config['use_pinecone'] ?? false);
	2033	+ // Check if Pinecone is enabled
	2034	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	2035	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
4185	2036
4186		- if ($use_pinecone && !empty($bot_pinecone_config['api_key'])) {
4187		- // Check Pinecone for this bot
4188		- $pinecone_data = $this->mxchat_get_pinecone_processed_content($bot_pinecone_config);
	2037	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2038	+ // ONLY check Pinecone if it's enabled
	2039	+ $pinecone_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
4189	2040	if (isset($pinecone_data[$post_id])) {
4190	2041	$is_update = true;
4191	2042	}
4192	2043	} else {
4193		- // Check WordPress DB (same as before since it's shared)
	2044	+ // ONLY check WordPress DB if Pinecone is not enabled
4194	2045	global $wpdb;
4195	2046	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
4196	2047	$existing_record = $wpdb->get_row($wpdb->prepare(
4197	2048	"SELECT id FROM $table_name WHERE source_url = %s",
		@@ -4201,54 +2052,54 @@
4201	2052	if ($existing_record) {
4202	2053	$is_update = true;
4203	2054	}
4204	2055	}
4205		-
4206		- // UPDATED 2.5.6: Determine content type based on post_type
4207		- $post_type = $post->post_type;
4208		- $content_type = 'content'; // Default fallback
4209		-
4210		- // Map WordPress post types to content types
4211		- switch ($post_type) {
4212		- case 'post':
4213		- $content_type = 'post';
4214		- break;
4215		- case 'page':
4216		- $content_type = 'page';
4217		- break;
4218		- case 'product':
4219		- $content_type = 'product';
4220		- break;
4221		- default:
4222		- // For custom post types, use the post type name
4223		- $content_type = sanitize_key($post_type);
4224		- break;
4225		- }
4226		-
4227		- // Use the centralized utility function with bot_id and content_type
	2056	+
	2057	+ // Use the centralized utility function for storage
4228	2058	$result = MxChat_Utils::submit_content_to_db(
4229		- $content,
4230		- $source_url,
	2059	+ $content,
	2060	+ $source_url,
4231	2061	$api_key,
4232		- $vector_id,
4233		- $bot_id,
4234		- $content_type
	2062	+ $vector_id
4235	2063	);
4236		-
	2064	+
4237	2065	if (is_wp_error($result)) {
4238		- MxChat_Admin::mxchat_log_debug('storage_error', 'Knowledge storage failed: ' . $result->get_error_message(), array('source_url' => $source_url));
4239	2066	wp_send_json_error('Storage failed: ' . $result->get_error_message());
4240	2067	exit;
4241	2068	}
4242		-
4243		- // Automatically apply role restriction based on tags
4244		- $this->apply_role_restriction_to_post($post_id, $source_url);
4245		-
	2069	+
	2070	+ // ================================
	2071	+ // UPDATE: Only update caches if Pinecone is enabled
	2072	+ // ================================
	2073	+
	2074	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2075	+ // Update vector ID cache for improved fetching
	2076	+ $this->mxchat_update_pinecone_vector_cache($vector_id);
	2077	+
	2078	+ // Update local processed content cache for immediate UI feedback
	2079	+ $pinecone_cache = get_option('mxchat_pinecone_processed_cache', array());
	2080	+ $pinecone_cache[$post_id] = array(
	2081	+ 'db_id' => $vector_id,
	2082	+ 'processed_date' => 'Just now',
	2083	+ 'url' => $source_url,
	2084	+ 'source' => 'pinecone',
	2085	+ 'timestamp' => current_time('timestamp')
	2086	+ );
	2087	+ update_option('mxchat_pinecone_processed_cache', $pinecone_cache);
	2088	+
	2089	+ // Also update the general processed content cache
	2090	+ $processed_cache = get_option('mxchat_processed_content_cache', array());
	2091	+ $processed_cache[$post_id] = array(
	2092	+ 'db_id' => $vector_id,
	2093	+ 'timestamp' => current_time('timestamp'),
	2094	+ 'url' => $source_url,
	2095	+ 'source' => 'pinecone'
	2096	+ );
	2097	+ update_option('mxchat_processed_content_cache', $processed_cache);
	2098	+ }
	2099	+
4246	2100	$operation_type = $is_update ? 'update' : 'new';
4247	2101
4248		- // Count ACF fields for debugging
4249		- $acf_field_count = count($acf_fields);
4250		-
4251	2102	// Success response with minimal data
4252	2103	wp_send_json_success(array(
4253	2104	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
4254	2105	'post_id' => $post_id,
		@@ -4253,95 +2104,27 @@
4253	2104	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
4254	2105	'post_id' => $post_id,
4255	2106	'title' => $post->post_title,
4256	2107	'operation_type' => $operation_type,
4257		- 'vector_id' => $vector_id,
4258		- 'acf_fields_found' => $acf_field_count,
4259		- 'pdf_extracted_count' => (int) $pdf_extracted_count,
4260		- 'content_preview' => substr($content, 0, 100) . '...',
4261		- 'bot_id' => $bot_id
	2108	+ 'vector_id' => $vector_id, // Include vector ID for debugging
	2109	+ 'cache_updated' => $use_pinecone // Indicate if cache was updated
4262	2110	));
4263	2111	exit;
4264	2112	}
4265	2113
4266		-private function apply_role_restriction_to_post($post_id, $source_url) {
4267		- // Get tag-role mappings
4268		- $mappings = get_option('mxchat_tag_role_mappings', array());
4269		-
4270		- if (empty($mappings)) {
4271		- return; // No mappings, leave as public
4272		- }
4273		-
4274		- // Get all tags for the post
4275		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
4276		-
4277		- if (empty($post_tags)) {
4278		- return; // No tags, leave as public
4279		- }
4280		-
4281		- // Determine the highest role restriction based on tags
4282		- $highest_role = 'public';
4283		- $role_hierarchy = array(
4284		- 'public' => 0,
4285		- 'logged_in' => 1,
4286		- 'subscriber' => 2,
4287		- 'contributor' => 3,
4288		- 'author' => 4,
4289		- 'editor' => 5,
4290		- 'administrator' => 6
4291		- );
4292		-
4293		- foreach ($post_tags as $tag_slug) {
4294		- if (isset($mappings[$tag_slug])) {
4295		- $role = $mappings[$tag_slug];
4296		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
4297		- $highest_role = $role;
4298		- }
4299		- }
4300		- }
4301		-
4302		- // If no restricted tags found, return (leave as public)
4303		- if ($highest_role === 'public') {
4304		- return;
4305		- }
4306		-
4307		- // Update the role restriction in the database
4308		- global $wpdb;
4309		-
4310		- // Check if using Pinecone
4311		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
4312		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
4313		-
4314		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
4315		- // Update Pinecone role restriction
4316		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
4317		- $vector_id = md5($source_url);
4318		-
4319		- $wpdb->replace(
4320		- $roles_table,
4321		- array(
4322		- 'vector_id' => $vector_id,
4323		- 'role_restriction' => $highest_role,
4324		- 'updated_at' => current_time('mysql')
4325		- ),
4326		- array('%s', '%s', '%s')
4327		- );
4328		- } else {
4329		- // Update WordPress DB
4330		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
4331		-
4332		- $wpdb->update(
4333		- $table_name,
4334		- array('role_restriction' => $highest_role),
4335		- array('source_url' => $source_url),
4336		- array('%s'),
4337		- array('%s')
4338		- );
4339		- }
4340		-}
4341	2114
	2115	+
	2116	+ /**
	2117	+ * Updates cache with new vector ID if absent
	2118	+ */
	2119	+ public function mxchat_update_pinecone_vector_cache($vector_id) {
	2120	+ $cached_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2121	+ if (!in_array($vector_id, $cached_ids)) {
	2122	+ $cached_ids[] = $vector_id;
	2123	+ update_option('mxchat_pinecone_vector_ids_cache', $cached_ids);
	2124	+ }
	2125	+ }
4342	2126	public function mxchat_get_public_post_types() {
4343		- // Get all public post types
4344	2127	$post_types = get_post_types(array('public' => true), 'objects');
4345	2128	$post_type_options = array();
4346	2129
4347	2130	foreach ($post_types as $post_type) {
		@@ -4347,45 +2130,51 @@
4347	2130	foreach ($post_types as $post_type) {
4348	2131	$post_type_options[$post_type->name] = $post_type->label;
4349	2132	}
4350	2133
4351		- // Also include common forum/community post types that might not be marked as public
4352		- $additional_types = array(
4353		- 'topic' => 'Forum Topics (bbPress)',
4354		- 'reply' => 'Forum Replies (bbPress)',
4355		- 'forum' => 'Forums (bbPress)',
4356		- 'wpforo_topic' => 'wpForo Topics',
4357		- 'wpforo_post' => 'wpForo Posts'
4358		- );
4359		-
4360		- foreach ($additional_types as $type_name => $type_label) {
4361		- if (post_type_exists($type_name) && !isset($post_type_options[$type_name])) {
4362		- $post_type_options[$type_name] = $type_label;
4363		- }
4364		- }
4365		-
4366	2134	return $post_type_options;
4367	2135	}
	2136	+public function mxchat_get_pinecone_processed_content($pinecone_options) {
	2137	+ //error_log('=== DEBUG: Starting mxchat_get_pinecone_processed_content ===');
	2138	+
	2139	+ // First check local cache for immediate updates
	2140	+ $cached_data = get_option('mxchat_pinecone_processed_cache', array());
	2141	+ //error_log('DEBUG: Found ' . count($cached_data) . ' items in local cache');
4368	2142
4369		-/**
4370		- * Retrieves processed content from Pinecone API
4371		- */
4372		-public function mxchat_get_pinecone_processed_content($pinecone_options) {
4373	2143	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4374	2144	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
	2145	+
	2146	+ //error_log('DEBUG: API key present: ' . (!empty($api_key) ? 'YES' : 'NO'));
	2147	+ //error_log('DEBUG: Host: ' . $host);
4375	2148
4376	2149	if (empty($api_key) || empty($host)) {
4377		- return array();
	2150	+ //error_log('DEBUG: Missing API credentials, returning cached data only');
	2151	+ return $cached_data;
4378	2152	}
4379	2153
4380	2154	$pinecone_data = array();
4381	2155
4382	2156	try {
4383		- // Always get fresh data from Pinecone
4384		- $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
	2157	+ // Method 1: Try to get vectors using cached vector IDs first
	2158	+ $cached_vector_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2159	+ //error_log('DEBUG: Found ' . count($cached_vector_ids) . ' cached vector IDs');
4385	2160
4386		- // Method 2: Final fallback - try stats endpoint (if available)
	2161	+ if (!empty($cached_vector_ids)) {
	2162	+ //error_log('DEBUG: Trying to fetch by cached vector IDs...');
	2163	+ $pinecone_data = $this->mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $cached_vector_ids);
	2164	+ //error_log('DEBUG: Fetch by IDs returned ' . count($pinecone_data) . ' items');
	2165	+ }
	2166	+
	2167	+ // Method 2: If no cached IDs or fetch failed, use scanning approach
4387	2168	if (empty($pinecone_data)) {
	2169	+ //error_log('DEBUG: Trying scanning approach...');
	2170	+ $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
	2171	+ //error_log('DEBUG: Scanning returned ' . count($pinecone_data) . ' items');
	2172	+ }
	2173	+
	2174	+ // Method 3: Final fallback - try stats endpoint
	2175	+ if (empty($pinecone_data)) {
	2176	+ //error_log('DEBUG: Trying stats endpoint...');
4388	2177	$stats_url = "https://{$host}/describe_index_stats";
4389	2178
4390	2179	$response = wp_remote_post($stats_url, array(
4391	2180	'headers' => array(
		@@ -4398,17 +2187,44 @@
4398	2187
4399	2188	if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
4400	2189	$body = wp_remote_retrieve_body($response);
4401	2190	$stats_data = json_decode($body, true);
	2191	+ //error_log('DEBUG: Pinecone stats: ' . print_r($stats_data, true));
	2192	+ } else {
	2193	+ if (is_wp_error($response)) {
	2194	+ //error_log('DEBUG: Stats endpoint error: ' . $response->get_error_message());
	2195	+ } else {
	2196	+ //error_log('DEBUG: Stats endpoint failed with code: ' . wp_remote_retrieve_response_code($response));
	2197	+ }
4402	2198	}
4403	2199	}
4404	2200
4405	2201	} catch (Exception $e) {
4406		- // Log error but return fresh data only
	2202	+ //error_log('DEBUG: Exception in get_pinecone_processed_content: ' . $e->getMessage());
4407	2203	}
4408	2204
4409		- return $pinecone_data;
	2205	+ // Merge cached data with Pinecone data
	2206	+ $merged_data = $pinecone_data;
	2207	+
	2208	+ foreach ($cached_data as $post_id => $cache_item) {
	2209	+ $cache_timestamp = $cache_item['timestamp'] ?? 0;
	2210	+ $time_diff = current_time('timestamp') - $cache_timestamp;
	2211	+
	2212	+ if ($time_diff < 300) { // 5 minutes = 300 seconds
	2213	+ $merged_data[$post_id] = $cache_item;
	2214	+ } else {
	2215	+ if (!isset($merged_data[$post_id])) {
	2216	+ $merged_data[$post_id] = $cache_item;
	2217	+ }
	2218	+ }
	2219	+ }
	2220	+
	2221	+ //error_log('DEBUG: Final merged data count: ' . count($merged_data));
	2222	+ //error_log('=== DEBUG: End mxchat_get_pinecone_processed_content ===');
	2223	+
	2224	+ return $merged_data;
4410	2225	}
	2226	+
4411	2227	public function mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $vector_ids) {
4412	2228	//error_log('=== DEBUG: Starting mxchat_fetch_pinecone_vectors_by_ids ===');
4413	2229
4414	2230	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
		@@ -4500,46 +2316,16 @@
4500	2316	return array();
4501	2317	}
4502	2318	}
4503	2319
4504		-/**
4505		- * Get embedding dimensions based on the selected model.
4506		- */
4507		-private function mxchat_get_embedding_dimensions() {
4508		- $options = get_option('mxchat_options', array());
4509		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4510		-
4511		- $model_dimensions = array(
4512		- 'text-embedding-ada-002' => 1536,
4513		- 'text-embedding-3-small' => 1536,
4514		- 'text-embedding-3-large' => 3072,
4515		- 'voyage-2' => 1024,
4516		- 'voyage-large-2' => 1536,
4517		- 'voyage-3-large' => 2048,
4518		- 'gemini-embedding-001' => 1536,
4519		- );
4520		-
4521		- if (strpos($selected_model, 'voyage-3-large') === 0) {
4522		- $custom_dimensions = $options['voyage_output_dimension'] ?? 2048;
4523		- return intval($custom_dimensions);
4524		- }
4525		-
4526		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4527		- $custom_dimensions = $options['gemini_output_dimension'] ?? 1536;
4528		- return intval($custom_dimensions);
4529		- }
4530		-
4531		- return $model_dimensions[$selected_model] ?? 1536;
4532		-}
4533		-
4534		-/**
4535		- * Scan Pinecone for processed content
4536		- */
4537	2320	public function mxchat_scan_pinecone_for_processed_content($pinecone_options) {
	2321	+ //error_log('=== DEBUG: Starting mxchat_scan_pinecone_for_processed_content ===');
	2322	+
4538	2323	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4539	2324	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
4540	2325
4541	2326	if (empty($api_key) || empty($host)) {
	2327	+ //error_log('DEBUG: Missing API credentials for scanning');
4542	2328	return array();
4543	2329	}
4544	2330
4545	2331	try {
		@@ -4546,18 +2332,17 @@
4546	2332	// Use multiple random vectors to get better coverage
4547	2333	$all_matches = array();
4548	2334	$seen_ids = array();
4549	2335
4550		- // Get correct dimensions for the configured embedding model
4551		- $dimensions = $this->mxchat_get_embedding_dimensions();
4552		-
4553	2336	// Try 3 different random vectors to get better coverage
4554	2337	for ($i = 0; $i < 3; $i++) {
	2338	+ //error_log('DEBUG: Scanning attempt ' . ($i + 1) . '/3');
	2339	+
4555	2340	$query_url = "https://{$host}/query";
4556	2341
4557	2342	// Generate a random unit vector instead of zeros
4558	2343	$random_vector = array();
4559		- for ($j = 0; $j < $dimensions; $j++) {
	2344	+ for ($j = 0; $j < 1536; $j++) {
4560	2345	$random_vector[] = (rand(-1000, 1000) / 1000.0);
4561	2346	}
4562	2347
4563	2348	// Normalize the vector to unit length
		@@ -4582,14 +2367,18 @@
4582	2367	'timeout' => 30
4583	2368	));
4584	2369
4585	2370	if (is_wp_error($response)) {
	2371	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' WP error: ' . $response->get_error_message());
4586	2372	continue;
4587	2373	}
4588	2374
4589	2375	$response_code = wp_remote_retrieve_response_code($response);
	2376	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' response code: ' . $response_code);
4590	2377
4591	2378	if ($response_code !== 200) {
	2379	+ $error_body = wp_remote_retrieve_body($response);
	2380	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' failed with body: ' . substr($error_body, 0, 500));
4592	2381	continue;
4593	2382	}
4594	2383
4595	2384	$body = wp_remote_retrieve_body($response);
		@@ -4595,8 +2384,9 @@
4595	2384	$body = wp_remote_retrieve_body($response);
4596	2385	$data = json_decode($body, true);
4597	2386
4598	2387	if (isset($data['matches'])) {
	2388	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' returned ' . count($data['matches']) . ' matches');
4599	2389	foreach ($data['matches'] as $match) {
4600	2390	$match_id = $match['id'] ?? '';
4601	2391	if (!empty($match_id) && !isset($seen_ids[$match_id])) {
4602	2392	$all_matches[] = $match;
		@@ -4602,14 +2392,18 @@
4602	2392	$all_matches[] = $match;
4603	2393	$seen_ids[$match_id] = true;
4604	2394	}
4605	2395	}
	2396	+ } else {
	2397	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' - no matches key in response');
4606	2398	}
4607	2399	}
4608	2400
4609		- // Convert matches to processed data format, grouping by URL to count chunks
	2401	+ //error_log('DEBUG: Total unique matches found: ' . count($all_matches));
	2402	+
	2403	+ // Convert matches to processed data format
4610	2404	$processed_data = array();
4611		- $url_chunk_counts = array();
	2405	+ $vector_ids_for_cache = array();
4612	2406
4613	2407	foreach ($all_matches as $match) {
4614	2408	$metadata = $match['metadata'] ?? array();
4615	2409	$source_url = $metadata['source_url'] ?? '';
		@@ -4617,14 +2411,8 @@
4617	2411
4618	2412	if (!empty($source_url) && !empty($match_id)) {
4619	2413	$post_id = url_to_postid($source_url);
4620	2414	if ($post_id) {
4621		- // Count chunks per post_id
4622		- if (!isset($url_chunk_counts[$post_id])) {
4623		- $url_chunk_counts[$post_id] = 0;
4624		- }
4625		- $url_chunk_counts[$post_id]++;
4626		-
4627	2415	$created_at = $metadata['created_at'] ?? '';
4628	2416	$processed_date = 'Recently';
4629	2417
4630	2418	if (!empty($created_at)) {
		@@ -4633,322 +2421,243 @@
4633	2421	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
4634	2422	}
4635	2423	}
4636	2424
4637		- // Only store if not already set, or update with newer timestamp
4638		- if (!isset($processed_data[$post_id]) ||
4639		- ($timestamp ?? 0) > ($processed_data[$post_id]['timestamp'] ?? 0)) {
4640		- $processed_data[$post_id] = array(
4641		- 'db_id' => $match_id,
4642		- 'processed_date' => $processed_date,
4643		- 'url' => $source_url,
4644		- 'source' => 'pinecone',
4645		- 'timestamp' => $timestamp ?? current_time('timestamp')
4646		- );
4647		- }
	2425	+ $processed_data[$post_id] = array(
	2426	+ 'db_id' => $match_id,
	2427	+ 'processed_date' => $processed_date,
	2428	+ 'url' => $source_url,
	2429	+ 'source' => 'pinecone',
	2430	+ 'timestamp' => $timestamp ?? current_time('timestamp')
	2431	+ );
	2432	+
	2433	+ $vector_ids_for_cache[] = $match_id;
4648	2434	}
4649	2435	}
4650	2436	}
4651	2437
4652		- // Add chunk counts to processed data
4653		- foreach ($url_chunk_counts as $post_id => $chunk_count) {
4654		- if (isset($processed_data[$post_id])) {
4655		- $processed_data[$post_id]['chunk_count'] = $chunk_count;
4656		- }
	2438	+ // Update the vector IDs cache for future use
	2439	+ if (!empty($vector_ids_for_cache)) {
	2440	+ update_option('mxchat_pinecone_vector_ids_cache', $vector_ids_for_cache);
	2441	+ //error_log('DEBUG: Updated vector IDs cache with ' . count($vector_ids_for_cache) . ' IDs');
4657	2442	}
4658	2443
	2444	+ //error_log('DEBUG: Returning ' . count($processed_data) . ' processed items from scanning');
4659	2445	return $processed_data;
4660	2446
4661	2447	} catch (Exception $e) {
	2448	+ //error_log('DEBUG: Exception in scan_pinecone_for_processed_content: ' . $e->getMessage());
4662	2449	return array();
4663	2450	}
4664	2451	}
4665		-/**
4666		- * Generate embeddings from input text for MXChat with bot support
4667		- */
4668		-private function mxchat_generate_embedding($text, $bot_id = 'default') {
4669		- // Enable detailed logging for debugging
4670		- //error_log('[MXCHAT-EMBED] Starting embedding generation for bot: ' . $bot_id . '. Text length: ' . strlen($text) . ' bytes');
4671		- //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4672	2452
4673		- // Get bot-specific options
4674		- $bot_options = $this->get_bot_options($bot_id);
4675		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	2453	+ /**
	2454	+ * Generates embeddings from input text for MXChat
	2455	+ */
	2456	+ private function mxchat_generate_embedding($text) {
	2457	+ // Enable detailed logging for debugging
	2458	+ //error_log('[MXCHAT-EMBED] Starting embedding generation. Text length: ' . strlen($text) . ' bytes');
	2459	+ //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4676	2460
4677		- // Opt-in: when the custom provider is selected for embeddings, index through
4678		- // the same custom endpoint the query path uses so stored vectors and query
4679		- // vectors share a model. Returns the vector array on success, or an error
4680		- // string on failure (this function's existing failure contract).
4681		- if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
4682		- if (!class_exists('MxChat_Utils')) {
4683		- require_once dirname(__FILE__) . '/../includes/class-mxchat-utils.php';
4684		- }
4685		- return MxChat_Utils::generate_embedding_custom($text, $options);
4686		- }
	2461	+ $options = get_option('mxchat_options');
	2462	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	2463	+ //error_log('[MXCHAT-EMBED] Selected embedding model: ' . $selected_model);
4687	2464
4688		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4689		- //error_log('[MXCHAT-EMBED] Selected embedding model for bot ' . $bot_id . ': ' . $selected_model);
	2465	+ // Determine provider and endpoint
	2466	+ if (strpos($selected_model, 'voyage') === 0) {
	2467	+ $api_key = $options['voyage_api_key'] ?? '';
	2468	+ $endpoint = 'https://api.voyageai.com/v1/embeddings';
	2469	+ $provider_name = 'Voyage AI';
	2470	+ //error_log('[MXCHAT-EMBED] Using Voyage AI API');
	2471	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	2472	+ $api_key = $options['gemini_api_key'] ?? '';
	2473	+ $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
	2474	+ $provider_name = 'Google Gemini';
	2475	+ //error_log('[MXCHAT-EMBED] Using Google Gemini API');
	2476	+ } else {
	2477	+ $api_key = $options['api_key'] ?? '';
	2478	+ $endpoint = 'https://api.openai.com/v1/embeddings';
	2479	+ $provider_name = 'OpenAI';
	2480	+ //error_log('[MXCHAT-EMBED] Using OpenAI API');
	2481	+ }
4690	2482
4691		- // Determine provider and endpoint
4692		- if (strpos($selected_model, 'voyage') === 0) {
4693		- $api_key = $options['voyage_api_key'] ?? '';
4694		- $endpoint = 'https://api.voyageai.com/v1/embeddings';
4695		- $provider_name = 'Voyage AI';
4696		- //error_log('[MXCHAT-EMBED] Using Voyage AI API for bot ' . $bot_id);
4697		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
4698		- $api_key = $options['gemini_api_key'] ?? '';
4699		- $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
4700		- $provider_name = 'Google Gemini';
4701		- //error_log('[MXCHAT-EMBED] Using Google Gemini API for bot ' . $bot_id);
4702		- } else {
4703		- $api_key = $options['api_key'] ?? '';
4704		- $endpoint = 'https://api.openai.com/v1/embeddings';
4705		- $provider_name = 'OpenAI';
4706		- //error_log('[MXCHAT-EMBED] Using OpenAI API for bot ' . $bot_id);
4707		- }
	2483	+ //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
4708	2484
4709		- //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
	2485	+ if (empty($api_key)) {
	2486	+ $error_message = sprintf('Missing %s API key. Please configure your API key in the MxChat settings.', $provider_name);
	2487	+ //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
	2488	+ return $error_message;
	2489	+ }
4710	2490
4711		- if (empty($api_key)) {
4712		- $error_message = sprintf('Missing %s API key for bot %s. Please configure your API key in the bot settings.', $provider_name, $bot_id);
4713		- //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
4714		- return $error_message;
4715		- }
	2491	+ // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
	2492	+ $estimated_tokens = ceil(str_word_count($text) / 0.75);
	2493	+ //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
4716	2494
4717		- // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
4718		- $estimated_tokens = ceil(str_word_count($text) / 0.75);
4719		- //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
	2495	+ if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
	2496	+ //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
	2497	+ // Consider truncating text here
	2498	+ }
4720	2499
4721		- if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
4722		- //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
4723		- // Consider truncating text here
4724		- }
	2500	+ // Prepare request body based on provider
	2501	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2502	+ // Gemini API format
	2503	+ $request_body = array(
	2504	+ 'model' => 'models/' . $selected_model,
	2505	+ 'content' => array(
	2506	+ 'parts' => array(
	2507	+ array('text' => $text)
	2508	+ )
	2509	+ )
	2510	+ );
4725	2511
4726		- // Prepare request body based on provider
4727		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4728		- // Gemini API format
4729		- $request_body = array(
4730		- 'model' => 'models/' . $selected_model,
4731		- 'content' => array(
4732		- 'parts' => array(
4733		- array('text' => $text)
4734		- )
4735		- )
4736		- );
	2512	+ // Set output dimensionality to 1536 for consistency with other models
	2513	+ $request_body['outputDimensionality'] = 1536;
	2514	+ } else {
	2515	+ // OpenAI/Voyage API format
	2516	+ $request_body = array(
	2517	+ 'model' => $selected_model,
	2518	+ 'input' => $text
	2519	+ );
4737	2520
4738		- // Set output dimensionality to 1536 for consistency with other models
4739		- $request_body['outputDimensionality'] = 1536;
4740		- } else {
4741		- // OpenAI/Voyage API format
4742		- $request_body = array(
4743		- 'model' => $selected_model,
4744		- 'input' => $text
4745		- );
	2521	+ // Add output_dimension for voyage-3-large model
	2522	+ if ($selected_model === 'voyage-3-large') {
	2523	+ $request_body['output_dimension'] = 2048;
	2524	+ }
	2525	+ }
4746	2526
4747		- // Add output_dimension for voyage-3-large model
4748		- if ($selected_model === 'voyage-3-large') {
4749		- $request_body['output_dimension'] = 2048;
4750		- }
4751		- }
	2527	+ //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
4752	2528
4753		- //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
	2529	+ // Prepare headers based on provider
	2530	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2531	+ // Gemini uses API key as query parameter
	2532	+ $endpoint .= '?key=' . $api_key;
	2533	+ $headers = array(
	2534	+ 'Content-Type' => 'application/json'
	2535	+ );
	2536	+ } else {
	2537	+ // OpenAI/Voyage use Bearer token
	2538	+ $headers = array(
	2539	+ 'Authorization' => 'Bearer ' . $api_key,
	2540	+ 'Content-Type' => 'application/json'
	2541	+ );
	2542	+ }
4754	2543
4755		- // Prepare headers based on provider
4756		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4757		- // Gemini uses API key as query parameter
4758		- $endpoint .= '?key=' . $api_key;
4759		- $headers = array(
4760		- 'Content-Type' => 'application/json'
4761		- );
4762		- } else {
4763		- // OpenAI/Voyage use Bearer token
4764		- $headers = array(
4765		- 'Authorization' => 'Bearer ' . $api_key,
4766		- 'Content-Type' => 'application/json'
4767		- );
4768		- }
	2544	+ // Make API request
	2545	+ //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
	2546	+ $response = wp_remote_post($endpoint, array(
	2547	+ 'body' => wp_json_encode($request_body),
	2548	+ 'headers' => $headers,
	2549	+ 'timeout' => 60 // Increased timeout for large inputs
	2550	+ ));
4769	2551
4770		- // Make API request
4771		- //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
4772		- $response = wp_remote_post($endpoint, array(
4773		- 'body' => wp_json_encode($request_body),
4774		- 'headers' => $headers,
4775		- 'timeout' => 60 // Increased timeout for large inputs
4776		- ));
	2552	+ // Handle wp_remote_post errors
	2553	+ if (is_wp_error($response)) {
	2554	+ $error_message = $response->get_error_message();
	2555	+ //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
	2556	+ return 'Connection error: ' . $error_message;
	2557	+ }
4777	2558
4778		- // Handle wp_remote_post errors
4779		- if (is_wp_error($response)) {
4780		- $error_message = $response->get_error_message();
4781		- //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
4782		- return 'Connection error: ' . $error_message;
4783		- }
	2559	+ // Get and check HTTP response code
	2560	+ $http_code = wp_remote_retrieve_response_code($response);
	2561	+ //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
4784	2562
4785		- // Get and check HTTP response code
4786		- $http_code = wp_remote_retrieve_response_code($response);
4787		- //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
	2563	+ if ($http_code !== 200) {
	2564	+ $error_body = wp_remote_retrieve_body($response);
	2565	+ //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
4788	2566
4789		- if ($http_code !== 200) {
4790		- $error_body = wp_remote_retrieve_body($response);
4791		- //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
	2567	+ // Try to parse error for more details
	2568	+ $error_json = json_decode($error_body, true);
	2569	+ if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
	2570	+ $error_type = $error_json['error']['type'] ?? 'unknown';
	2571	+ $error_message = $error_json['error']['message'] ?? 'No message';
	2572	+ //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
	2573	+ //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
4792	2574
4793		- // Try to parse error for more details
4794		- $error_json = json_decode($error_body, true);
4795		- if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
4796		- $error_type = $error_json['error']['type'] ?? 'unknown';
4797		- $error_message = $error_json['error']['message'] ?? 'No message';
4798		- //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
4799		- //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
	2575	+ // Customize error message for common API errors
	2576	+ if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
	2577	+ $error_message = sprintf('Invalid %s API key. Please check your API key in the MxChat settings.', $provider_name);
	2578	+ } elseif ($error_type === 'authentication_error') {
	2579	+ $error_message = sprintf('%s authentication failed. Please verify your API key in the MxChat settings.', $provider_name);
	2580	+ }
4800	2581
4801		- // Customize error message for common API errors
4802		- if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
4803		- $error_message = sprintf('Invalid %s API key for bot %s. Please check your API key in the bot settings.', $provider_name, $bot_id);
4804		- } elseif ($error_type === 'authentication_error') {
4805		- $error_message = sprintf('%s authentication failed for bot %s. Please verify your API key in the bot settings.', $provider_name, $bot_id);
4806		- }
	2582	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2583	+ return $error_message;
	2584	+ }
4807	2585
4808		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4809		- return $error_message;
4810		- }
	2586	+ $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding", $http_code);
	2587	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2588	+ return $error_message;
	2589	+ }
4811	2590
4812		- $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding for bot %s", $http_code, $bot_id);
4813		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4814		- return $error_message;
4815		- }
	2591	+ // Parse response body
	2592	+ $response_body = wp_remote_retrieve_body($response);
	2593	+ //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
4816	2594
4817		- // Parse response body
4818		- $response_body = wp_remote_retrieve_body($response);
4819		- //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
	2595	+ $response_data = json_decode($response_body, true);
4820	2596
4821		- $response_data = json_decode($response_body, true);
	2597	+ if (json_last_error() !== JSON_ERROR_NONE) {
	2598	+ $error = json_last_error_msg();
	2599	+ //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
	2600	+ //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
	2601	+ return "Failed to parse API response: $error";
	2602	+ }
4822	2603
4823		- if (json_last_error() !== JSON_ERROR_NONE) {
4824		- $error = json_last_error_msg();
4825		- //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
4826		- //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
4827		- return "Failed to parse API response: $error";
4828		- }
	2604	+ // Handle different response formats based on provider
	2605	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2606	+ // Gemini API response format
	2607	+ if (isset($response_data['embedding']['values'])) {
	2608	+ $embedding_dimensions = count($response_data['embedding']['values']);
	2609	+ //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
4829	2610
4830		- // Handle different response formats based on provider
4831		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4832		- // Gemini API response format
4833		- if (isset($response_data['embedding']['values'])) {
4834		- $embedding_dimensions = count($response_data['embedding']['values']);
4835		- //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
	2611	+ // Check if embedding dimensions are as expected (should be 1536)
	2612	+ if ($embedding_dimensions !== 1536) {
	2613	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
	2614	+ }
4836	2615
4837		- // Check if embedding dimensions are as expected (should be 1536)
4838		- if ($embedding_dimensions !== 1536) {
4839		- //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
4840		- }
	2616	+ return $response_data['embedding']['values'];
	2617	+ } else {
	2618	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
	2619	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4841	2620
4842		- return $response_data['embedding']['values'];
4843		- } else {
4844		- //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
4845		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2621	+ if (isset($response_data['error'])) {
	2622	+ $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
	2623	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2624	+ return $error_message;
	2625	+ }
4846	2626
4847		- if (isset($response_data['error'])) {
4848		- $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
4849		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4850		- return $error_message;
4851		- }
	2627	+ $error_message = "Invalid Gemini API response format: No embedding found";
	2628	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2629	+ return $error_message;
	2630	+ }
	2631	+ } else {
	2632	+ // OpenAI/Voyage API response format
	2633	+ if (isset($response_data['data'][0]['embedding'])) {
	2634	+ $embedding_dimensions = count($response_data['data'][0]['embedding']);
	2635	+ //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
4852	2636
4853		- $error_message = "Invalid Gemini API response format: No embedding found";
4854		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4855		- return $error_message;
4856		- }
4857		- } else {
4858		- // OpenAI/Voyage API response format
4859		- if (isset($response_data['data'][0]['embedding'])) {
4860		- $embedding_dimensions = count($response_data['data'][0]['embedding']);
4861		- //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
	2637	+ // Check if embedding dimensions are as expected
	2638	+ if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
	2639	+ ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
	2640	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
	2641	+ }
4862	2642
4863		- // Check if embedding dimensions are as expected
4864		- if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
4865		- ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
4866		- //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
4867		- }
	2643	+ return $response_data['data'][0]['embedding'];
	2644	+ } else {
	2645	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
	2646	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4868	2647
4869		- return $response_data['data'][0]['embedding'];
4870		- } else {
4871		- //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
4872		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2648	+ if (isset($response_data['error'])) {
	2649	+ $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
	2650	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2651	+ return $error_message;
	2652	+ }
4873	2653
4874		- if (isset($response_data['error'])) {
4875		- $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
4876		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4877		- return $error_message;
4878		- }
4879		-
4880		- $error_message = "Invalid API response format: No embedding found";
4881		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4882		- return $error_message;
4883		- }
4884		- }
4885		-}
4886		-
4887		-/**
4888		- * Get bot-specific options for multi-bot functionality
4889		- * Falls back to default options if bot_id is 'default' or multi-bot add-on is not active
4890		- */
4891		-private function get_bot_options($bot_id = 'default') {
4892		- //error_log("MXCHAT DEBUG: get_bot_options called for bot: " . $bot_id);
4893		-
4894		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4895		- //error_log("MXCHAT DEBUG: Using default options (no multi-bot or bot is 'default')");
4896		- return array();
4897		- }
4898		-
4899		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
4900		-
4901		- if (!empty($bot_options)) {
4902		- //error_log("MXCHAT DEBUG: Got bot-specific options from filter");
4903		- if (isset($bot_options['similarity_threshold'])) {
4904		- //error_log(" - similarity_threshold: " . $bot_options['similarity_threshold']);
4905		- }
4906		- }
4907		-
4908		- return is_array($bot_options) ? $bot_options : array();
4909		-}
4910		-
4911		-/**
4912		- * Get bot-specific Pinecone configuration
4913		- * Used in the knowledge retrieval functions
4914		- */
4915		-// Also add debugging to your get_bot_pinecone_config function
4916		-private function get_bot_pinecone_config($bot_id = 'default') {
4917		- //error_log("MXCHAT DEBUG: get_bot_pinecone_config called for bot: " . $bot_id);
4918		-
4919		- // If default bot or multi-bot add-on not active, use default Pinecone config
4920		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4921		- //error_log("MXCHAT DEBUG: Using default Pinecone config (no multi-bot or bot is 'default')");
4922		- $addon_options = get_option('mxchat_pinecone_addon_options', array());
4923		- $config = array(
4924		- 'use_pinecone' => (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1'),
4925		- 'api_key' => $addon_options['mxchat_pinecone_api_key'] ?? '',
4926		- 'host' => $addon_options['mxchat_pinecone_host'] ?? '',
4927		- 'namespace' => $addon_options['mxchat_pinecone_namespace'] ?? ''
4928		- );
4929		- //error_log("MXCHAT DEBUG: Default config - use_pinecone: " . ($config['use_pinecone'] ? 'true' : 'false'));
4930		- return $config;
4931		- }
4932		-
4933		- //error_log("MXCHAT DEBUG: Calling filter 'mxchat_get_bot_pinecone_config' for bot: " . $bot_id);
4934		-
4935		- // Hook for multi-bot add-on to provide bot-specific Pinecone config
4936		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
4937		-
4938		- if (!empty($bot_pinecone_config)) {
4939		- //error_log("MXCHAT DEBUG: Got bot-specific config from filter");
4940		- //error_log(" - use_pinecone: " . (isset($bot_pinecone_config['use_pinecone']) ? ($bot_pinecone_config['use_pinecone'] ? 'true' : 'false') : 'not set'));
4941		- //error_log(" - host: " . ($bot_pinecone_config['host'] ?? 'not set'));
4942		- //error_log(" - namespace: " . ($bot_pinecone_config['namespace'] ?? 'not set'));
4943		- } else {
4944		- //error_log("MXCHAT DEBUG: Filter returned empty config!");
4945		- }
4946		-
4947		- return is_array($bot_pinecone_config) ? $bot_pinecone_config : array();
4948		-}
4949		-
4950		-
	2654	+ $error_message = "Invalid API response format: No embedding found";
	2655	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2656	+ return $error_message;
	2657	+ }
	2658	+ }
	2659	+ }
4951	2660	public function mxchat_ajax_dismiss_completed_status() {
4952	2661	try {
4953	2662	// Verify the request
4954	2663	check_ajax_referer('mxchat_status_nonce', 'nonce');
		@@ -5311,564 +3020,18 @@
5311	3020
5312	3021	return $html;
5313	3022	}
5314	3023
5315		-/**
5316		- * Get all ACF fields for a specific post, excluding any fields the user has disabled
5317		- */
5318		-public function mxchat_get_acf_fields_for_post($post_id) {
5319		- if (!function_exists('get_fields')) {
5320		- return array();
5321		- }
5322		-
5323		- $fields = get_fields($post_id);
5324		- if (!$fields || !is_array($fields)) {
5325		- return array();
5326		- }
5327		-
5328		- // Get excluded fields from settings
5329		- $excluded_fields = get_option('mxchat_acf_excluded_fields', array());
5330		- if (!empty($excluded_fields) && is_array($excluded_fields)) {
5331		- foreach ($excluded_fields as $excluded_field) {
5332		- if (isset($fields[$excluded_field])) {
5333		- unset($fields[$excluded_field]);
5334		- }
5335		- }
5336		- }
5337		-
5338		- return $fields;
5339		-}
5340		-
5341		-/**
5342		- * Get all registered ACF field groups and their fields for the settings UI
5343		- */
5344		-public function mxchat_get_all_acf_fields() {
5345		- if (!function_exists('acf_get_field_groups') || !function_exists('acf_get_fields')) {
5346		- return array();
5347		- }
5348		-
5349		- $all_fields = array();
5350		- $field_groups = acf_get_field_groups();
5351		-
5352		- if (!empty($field_groups)) {
5353		- foreach ($field_groups as $group) {
5354		- $group_fields = acf_get_fields($group['key']);
5355		- if (!empty($group_fields)) {
5356		- $all_fields[$group['title']] = array();
5357		- foreach ($group_fields as $field) {
5358		- $all_fields[$group['title']][] = array(
5359		- 'name' => $field['name'],
5360		- 'label' => $field['label'],
5361		- 'type' => $field['type']
5362		- );
5363		- }
5364		- }
5365		- }
5366		- }
5367		-
5368		- return $all_fields;
5369		-}
5370		-
5371		-/**
5372		- * Get whitelisted custom post meta for a given post
5373		- * This allows non-ACF meta fields (like OptionTree, theme meta boxes) to be included in embeddings
5374		- */
5375		-public function mxchat_get_whitelisted_post_meta($post_id) {
5376		- $whitelist = get_option('mxchat_custom_meta_whitelist', '');
5377		-
5378		- if (empty($whitelist)) {
5379		- return array();
5380		- }
5381		-
5382		- // Parse the whitelist - one meta key per line
5383		- $meta_keys = array_filter(array_map('trim', explode("\n", $whitelist)));
5384		-
5385		- if (empty($meta_keys)) {
5386		- return array();
5387		- }
5388		-
5389		- $result = array();
5390		-
5391		- foreach ($meta_keys as $key) {
5392		- // Skip empty keys
5393		- if (empty($key)) {
5394		- continue;
5395		- }
5396		-
5397		- $value = get_post_meta($post_id, $key, true);
5398		-
5399		- // Only include non-empty string values
5400		- if (!empty($value) && is_string($value)) {
5401		- $result[$key] = $value;
5402		- } elseif (!empty($value) && is_array($value)) {
5403		- // Handle array values by joining them
5404		- $flat_value = $this->mxchat_flatten_meta_array($value);
5405		- if (!empty($flat_value)) {
5406		- $result[$key] = $flat_value;
5407		- }
5408		- }
5409		- }
5410		-
5411		- return $result;
5412		-}
5413		-
5414		-/**
5415		- * Flatten array meta values into a readable string
5416		- */
5417		-private function mxchat_flatten_meta_array($array, $depth = 0) {
5418		- if ($depth > 3) {
5419		- return ''; // Prevent infinite recursion
5420		- }
5421		-
5422		- $parts = array();
5423		-
5424		- foreach ($array as $key => $value) {
5425		- if (is_string($value) && !empty($value)) {
5426		- $parts[] = $value;
5427		- } elseif (is_array($value)) {
5428		- $nested = $this->mxchat_flatten_meta_array($value, $depth + 1);
5429		- if (!empty($nested)) {
5430		- $parts[] = $nested;
5431		- }
5432		- }
5433		- }
5434		-
5435		- return implode(', ', $parts);
5436		-}
5437		-
5438		-/**
5439		- * Format ACF field values for content extraction
5440		- */
5441		-public function mxchat_format_acf_field_value($value, $field_name = '', $post_id = 0) {
5442		- if (empty($value)) {
5443		- return '';
5444		- }
5445		-
5446		- // Handle WP_Post objects first (THIS IS THE KEY FIX)
5447		- if ($value instanceof WP_Post) {
5448		- return $value->post_title ?: '';
5449		- }
5450		-
5451		- // Handle other WP objects
5452		- if (is_object($value)) {
5453		- if (isset($value->post_title)) {
5454		- return $value->post_title;
5455		- } elseif (isset($value->display_name)) {
5456		- return $value->display_name;
5457		- } elseif (isset($value->name)) {
5458		- return $value->name;
5459		- } elseif (method_exists($value, '__toString')) {
5460		- try {
5461		- return (string) $value;
5462		- } catch (Exception $e) {
5463		- return '';
5464		- }
5465		- }
5466		- // For any other objects, return empty string
5467		- return '';
5468		- }
5469		-
5470		- // Handle different ACF field types
5471		- if (is_array($value)) {
5472		- // Check if it's an image/file field
5473		- if (isset($value['url'])) {
5474		- // Image field - return alt text, title, or caption
5475		- if (!empty($value['alt'])) {
5476		- return $value['alt'];
5477		- } elseif (!empty($value['title'])) {
5478		- return $value['title'];
5479		- } elseif (!empty($value['caption'])) {
5480		- return $value['caption'];
5481		- } else {
5482		- return ''; // Don't include just the URL
5483		- }
5484		- }
5485		-
5486		- // Check if it's a post object or relationship field
5487		- if (isset($value['post_title'])) {
5488		- return $value['post_title'];
5489		- }
5490		-
5491		- // Check if it's a user field
5492		- if (isset($value['display_name'])) {
5493		- return $value['display_name'];
5494		- }
5495		-
5496		- // Check if it's a taxonomy term
5497		- if (isset($value['name']) && isset($value['taxonomy'])) {
5498		- return $value['name'];
5499		- }
5500		-
5501		- // Check if it's a select field with label
5502		- if (isset($value['label'])) {
5503		- return $value['label'];
5504		- }
5505		-
5506		- // Check for repeater field or flexible content
5507		- if (is_numeric(key($value))) {
5508		- $sub_values = array();
5509		- foreach ($value as $sub_item) {
5510		- if (is_array($sub_item)) {
5511		- // For repeater/flexible content, extract text values
5512		- $sub_text = $this->mxchat_extract_text_from_acf_array($sub_item);
5513		- if (!empty($sub_text)) {
5514		- $sub_values[] = $sub_text;
5515		- }
5516		- } elseif ($sub_item instanceof WP_Post) {
5517		- // Handle WP_Post objects in arrays
5518		- $sub_values[] = $sub_item->post_title ?: '';
5519		- } else {
5520		- $sub_values[] = (string) $sub_item;
5521		- }
5522		- }
5523		- return implode(', ', array_filter($sub_values));
5524		- }
5525		-
5526		- // For other arrays, try to extract meaningful text
5527		- $text_values = array();
5528		- foreach ($value as $key => $val) {
5529		- if (is_string($val) && !empty(trim($val))) {
5530		- $text_values[] = trim($val);
5531		- } elseif ($val instanceof WP_Post) {
5532		- // Handle WP_Post objects in associative arrays
5533		- $text_values[] = $val->post_title ?: '';
5534		- } elseif (is_array($val) && isset($val['post_title'])) {
5535		- $text_values[] = $val['post_title'];
5536		- } elseif (is_array($val) && isset($val['name'])) {
5537		- $text_values[] = $val['name'];
5538		- }
5539		- }
5540		-
5541		- return implode(', ', array_filter($text_values));
5542		- }
5543		-
5544		- // Handle boolean values
5545		- if (is_bool($value)) {
5546		- return $value ? 'Yes' : 'No';
5547		- }
5548		-
5549		- // Handle numeric values
5550		- if (is_numeric($value)) {
5551		- return (string) $value;
5552		- }
5553		-
5554		- // Handle string values
5555		- if (is_string($value)) {
5556		- return trim($value);
5557		- }
5558		-
5559		- // For anything else that we can't handle, return empty string
5560		- // This prevents the "Object could not be converted to string" error
5561		- return '';
5562		-}
5563		-
5564		-/**
5565		- * Extract text from complex ACF array structures
5566		- */
5567		-private function mxchat_extract_text_from_acf_array($array) {
5568		- if (!is_array($array)) {
5569		- return '';
5570		- }
5571		-
5572		- $text_parts = array();
5573		-
5574		- foreach ($array as $key => $value) {
5575		- if (is_string($value) && !empty(trim($value))) {
5576		- // Skip keys that are likely to be IDs or technical values
5577		- if (!is_numeric($value) || strlen($value) > 10) {
5578		- $text_parts[] = trim($value);
5579		- }
5580		- } elseif ($value instanceof WP_Post) {
5581		- // Handle WP_Post objects
5582		- $text_parts[] = $value->post_title ?: '';
5583		- } elseif (is_array($value)) {
5584		- if (isset($value['post_title'])) {
5585		- $text_parts[] = $value['post_title'];
5586		- } elseif (isset($value['name'])) {
5587		- $text_parts[] = $value['name'];
5588		- } elseif (isset($value['label'])) {
5589		- $text_parts[] = $value['label'];
5590		- }
5591		- } elseif (is_object($value)) {
5592		- // Handle other objects safely
5593		- if (isset($value->post_title)) {
5594		- $text_parts[] = $value->post_title;
5595		- } elseif (isset($value->name)) {
5596		- $text_parts[] = $value->name;
5597		- } elseif (isset($value->display_name)) {
5598		- $text_parts[] = $value->display_name;
5599		- }
5600		- }
5601		- }
5602		-
5603		- return implode(', ', array_filter($text_parts));
5604		-}
5605		-
5606		-/**
5607		- * Walk an ACF field value tree and collect attachment IDs for any value that
5608		- * resolves to a PDF in the WordPress media library. Handles the three shapes
5609		- * ACF returns for File/Image/URL fields (array with ID+url, integer attachment ID,
5610		- * plain URL string), and recurses through repeater/group/flexible content.
5611		- *
5612		- * @param mixed $value The ACF field value (any depth)
5613		- * @param array $out Accumulator (passed by reference) for attachment IDs
5614		- * @param int $depth Recursion guard
5615		- */
5616		-private function mxchat_collect_pdf_attachment_ids_from_acf_value($value, &$out, $depth = 0) {
5617		- if ($depth > 6) {
5618		- return; // prevent runaway recursion on circular/very-deep structures
5619		- }
5620		-
5621		- if (empty($value)) {
5622		- return;
5623		- }
5624		-
5625		- // Array shapes: ACF File/Image return value=array; repeaters/groups are arrays of arrays
5626		- if (is_array($value)) {
5627		- // Direct File/Image-style array (has 'url' and usually 'ID' + 'mime_type')
5628		- $looks_like_attachment = isset($value['url']) || isset($value['ID']) || isset($value['id']);
5629		- if ($looks_like_attachment) {
5630		- $att_id = 0;
5631		- if (!empty($value['ID']) && is_numeric($value['ID'])) {
5632		- $att_id = (int) $value['ID'];
5633		- } elseif (!empty($value['id']) && is_numeric($value['id'])) {
5634		- $att_id = (int) $value['id'];
5635		- } elseif (!empty($value['url']) && is_string($value['url'])) {
5636		- $att_id = (int) attachment_url_to_postid($value['url']);
5637		- }
5638		-
5639		- $is_pdf = false;
5640		- if (!empty($value['mime_type']) && $value['mime_type'] === 'application/pdf') {
5641		- $is_pdf = true;
5642		- } elseif (!empty($value['subtype']) && strtolower((string) $value['subtype']) === 'pdf') {
5643		- $is_pdf = true;
5644		- } elseif (!empty($value['url']) && is_string($value['url']) && $this->mxchat_url_looks_like_pdf($value['url'])) {
5645		- $is_pdf = true;
5646		- } elseif ($att_id && get_post_mime_type($att_id) === 'application/pdf') {
5647		- $is_pdf = true;
5648		- }
5649		-
5650		- if ($is_pdf && $att_id && get_post_mime_type($att_id) === 'application/pdf') {
5651		- $out[] = $att_id;
5652		- }
5653		- // An array node that represents one attachment doesn't contain other
5654		- // attachments inside it — done with this branch.
5655		- return;
5656		- }
5657		-
5658		- // Recurse: repeater rows, flexible-content layouts, groups, etc.
5659		- foreach ($value as $sub) {
5660		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($sub, $out, $depth + 1);
5661		- }
5662		- return;
5663		- }
5664		-
5665		- // Plain numeric attachment ID (ACF File field set to "Return: ID")
5666		- if (is_numeric($value)) {
5667		- $att_id = (int) $value;
5668		- if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5669		- $out[] = $att_id;
5670		- }
5671		- return;
5672		- }
5673		-
5674		- // Plain string — URL pointing at a PDF (ACF File field set to "Return: URL", or a custom URL/text field)
5675		- if (is_string($value)) {
5676		- $trimmed = trim($value);
5677		- if ($trimmed !== '' && $this->mxchat_url_looks_like_pdf($trimmed)) {
5678		- $att_id = (int) attachment_url_to_postid($trimmed);
5679		- if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5680		- $out[] = $att_id;
5681		- }
5682		- }
5683		- return;
5684		- }
5685		-}
5686		-
5687		-/**
5688		- * Heuristic: does this URL/string look like a PDF reference?
5689		- * Tolerates query strings and fragments (#page=2).
5690		- */
5691		-private function mxchat_url_looks_like_pdf($url) {
5692		- if (!is_string($url) || $url === '') {
5693		- return false;
5694		- }
5695		- // Strip query + fragment before checking extension
5696		- $path = preg_replace('/[?#].*$/', '', $url);
5697		- return (bool) preg_match('/\.pdf$/i', $path);
5698		-}
5699		-
5700		-/**
5701		- * Extract text from a PDF attachment by ID using the bundled Smalot parser.
5702		- * Reads the file directly from disk via get_attached_file (no HTTP fetch).
5703		- * Result is cached on the attachment as post_meta keyed by file mtime so we
5704		- * only parse the same PDF once unless the file changes on disk.
5705		- *
5706		- * @param int $attachment_id
5707		- * @return string Extracted plain text, or '' on failure.
5708		- */
5709		-private function mxchat_extract_pdf_text_by_attachment_id($attachment_id) {
5710		- $attachment_id = (int) $attachment_id;
5711		- if ($attachment_id <= 0) {
5712		- return '';
5713		- }
5714		- if (get_post_mime_type($attachment_id) !== 'application/pdf') {
5715		- return '';
5716		- }
5717		-
5718		- $pdf_path = get_attached_file($attachment_id);
5719		- if (empty($pdf_path) || !file_exists($pdf_path) || !is_readable($pdf_path)) {
5720		- return '';
5721		- }
5722		-
5723		- // Raw-file size cap. Parsing very large PDFs can OOM the request; skip with a log entry
5724		- // and let the rest of the ACF content land in the KB. Filterable for users who need it bigger.
5725		- $default_max_bytes = 25 * 1024 * 1024;
5726		- $max_bytes = (int) apply_filters('mxchat_acf_pdf_max_bytes', $default_max_bytes, $attachment_id, $pdf_path);
5727		- if ($max_bytes > 0) {
5728		- $file_size = @filesize($pdf_path);
5729		- if ($file_size !== false && $file_size > $max_bytes) {
5730		- error_log(sprintf(
5731		- '[mxchat] ACF PDF skipped (over size cap): attachment %d "%s" %d bytes > cap %d',
5732		- $attachment_id,
5733		- basename($pdf_path),
5734		- $file_size,
5735		- $max_bytes
5736		- ));
5737		- return '';
5738		- }
5739		- }
5740		-
5741		- $mtime = @filemtime($pdf_path);
5742		- $cache_meta_key = '_mxchat_acf_pdf_text_v1';
5743		- $cached = get_post_meta($attachment_id, $cache_meta_key, true);
5744		- if (is_array($cached) && isset($cached['mtime'], $cached['text']) && (int) $cached['mtime'] === (int) $mtime) {
5745		- return (string) $cached['text'];
5746		- }
5747		-
5748		- $text = '';
5749		- try {
5750		- if (function_exists('mxchat_load_pdf_parser')) {
5751		- mxchat_load_pdf_parser();
5752		- }
5753		- if (!class_exists('\\Smalot\\PdfParser\\Parser')) {
5754		- return '';
5755		- }
5756		- $parser = new \Smalot\PdfParser\Parser();
5757		- $pdf = $parser->parseFile($pdf_path);
5758		- $pages = $pdf->getPages();
5759		- $page_texts = array();
5760		- foreach ($pages as $page) {
5761		- $page_text = '';
5762		- try {
5763		- $page_text = $page->getText();
5764		- } catch (\Exception $e) {
5765		- $page_text = '';
5766		- }
5767		- if (!empty($page_text)) {
5768		- $page_texts[] = $page_text;
5769		- }
5770		- }
5771		- $text = trim(implode("\n\n", $page_texts));
5772		- } catch (\Exception $e) {
5773		- error_log('[mxchat] ACF PDF extraction failed for attachment ' . $attachment_id . ': ' . $e->getMessage());
5774		- return '';
5775		- } catch (\Throwable $e) {
5776		- error_log('[mxchat] ACF PDF extraction error for attachment ' . $attachment_id . ': ' . $e->getMessage());
5777		- return '';
5778		- }
5779		-
5780		- // Cap per-PDF text to avoid blowing up the embedding payload on enormous PDFs.
5781		- // The chunker downstream will still split this into multiple vectors.
5782		- $max_len = (int) apply_filters('mxchat_acf_pdf_text_max_length', 50000);
5783		- if ($max_len > 0 && strlen($text) > $max_len) {
5784		- $text = substr($text, 0, $max_len);
5785		- }
5786		-
5787		- update_post_meta($attachment_id, $cache_meta_key, array(
5788		- 'mtime' => (int) $mtime,
5789		- 'text' => $text,
5790		- ));
5791		-
5792		- return $text;
5793		-}
5794		-
5795		-/**
5796		- * Handle ACF save - fires after ACF fields are saved
5797		- * This ensures ACF field data is available when syncing to knowledge base
5798		- */
5799		-public function mxchat_handle_acf_save($post_id) {
5800		- // Skip if not a valid post
5801		- if (!$post_id || $post_id === 'options') {
5802		- return;
5803		- }
5804		-
5805		- // Skip autosaves and revisions
	3024	+public function mxchat_handle_post_update($post_id, $post, $update) {
	3025	+ // Basic validation checks
5806	3026	if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5807	3027	return;
5808	3028	}
5809		-
5810		- $post = get_post($post_id);
5811		- if (!$post) {
5812		- return;
5813		- }
5814		-
5815		- $post_type = $post->post_type;
5816		-
5817		- // Check if sync is enabled for this post type
5818		- $should_sync = false;
5819		-
5820		- if ($post_type === 'post' && get_option('mxchat_auto_sync_posts') === '1') {
5821		- $should_sync = true;
5822		- } else if ($post_type === 'page' && get_option('mxchat_auto_sync_pages') === '1') {
5823		- $should_sync = true;
5824		- } else if ($post_type === 'product' && class_exists('WooCommerce')) {
5825		- // WooCommerce products - check if WooCommerce integration is enabled
5826		- $options = get_option('mxchat_options', array());
5827		- if (isset($options['enable_woocommerce_integration']) &&
5828		- ($options['enable_woocommerce_integration'] === '1' || $options['enable_woocommerce_integration'] === 'on')) {
5829		- $should_sync = true;
5830		- }
5831		- } else {
5832		- // Check custom post types
5833		- $option_name = 'mxchat_auto_sync_' . $post_type;
5834		- if (get_option($option_name) === '1') {
5835		- $should_sync = true;
5836		- }
5837		- }
5838		-
5839		- if (!$should_sync) {
5840		- return;
5841		- }
5842		-
5843		- // Only process published posts
	3029	+
	3030	+ // Only process published content
5844	3031	if ($post->post_status !== 'publish') {
5845	3032	return;
5846	3033	}
5847		-
5848		- // Check if this post has any ACF fields - if not, no need to re-sync
5849		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
5850		- if (empty($acf_fields)) {
5851		- return;
5852		- }
5853		-
5854		- // Use a transient to prevent duplicate processing (post_updated may have already run)
5855		- $transient_key = 'mxchat_acf_synced_' . $post_id;
5856		- if (get_transient($transient_key)) {
5857		- return;
5858		- }
5859		- set_transient($transient_key, true, 60); // Prevent re-processing for 60 seconds
5860		-
5861		- // Re-run the sync with ACF data now available
5862		- // We pass $update=true since this is effectively an update with ACF data
5863		- $this->mxchat_handle_post_update($post_id, $post, true);
5864		-}
5865		-
5866		-public function mxchat_handle_post_update($post_id, $post, $update) {
5867		- // Basic validation checks
5868		- if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5869		- return;
5870		- }
5871	3034
5872	3035	$post_type = $post->post_type;
5873	3036
5874	3037	// Check if sync is enabled for this post type
		@@ -5890,276 +3053,80 @@
5890	3053	if (!$should_sync) {
5891	3054	return;
5892	3055	}
5893	3056
5894		- // Check if we have stored the previous status and URL in our transients
5895		- $previous_status_key = 'mxchat_prev_status_' . $post_id;
5896		- $previous_status = get_transient($previous_status_key);
	3057	+ // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
	3058	+ $title = get_the_title($post_id);
	3059	+ $content = get_post_field('post_content', $post_id);
5897	3060
5898		- $previous_url_key = 'mxchat_prev_url_' . $post_id;
5899		- $previous_url = get_transient($previous_url_key);
	3061	+ // Apply WordPress content filters to get properly formatted content
	3062	+ $content = apply_filters('the_content', $content);
5900	3063
5901		- // If the post was previously published but is now not published, remove from knowledge base
5902		- if ($previous_status === 'publish' && $post->post_status !== 'publish') {
5903		- // Use the stored URL from when it was published, or fall back to current permalink
5904		- $source_url = $previous_url ?: get_permalink($post_id);
5905		-
5906		- if ($source_url) {
5907		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
5908		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	3064	+ // Strip tags but preserve structure
	3065	+ $content = wp_strip_all_tags($content);
	3066	+
	3067	+ // Combine title and content
	3068	+ $final_content = $title . "\n\n" . $content;
	3069	+
	3070	+ // For custom post types like job_listing, include additional fields
	3071	+ if ($post_type === 'job_listing') {
	3072	+ // Add job-specific meta if available
	3073	+ $job_location = get_post_meta($post_id, '_job_location', true);
	3074	+ if (!empty($job_location)) {
	3075	+ $final_content .= "\n\nLocation: " . $job_location;
5909	3076	}
5910		-
5911		- // Clean up the transients and exit early
5912		- delete_transient($previous_status_key);
5913		- delete_transient($previous_url_key);
5914		- return;
5915		- }
5916		-
5917		- // Slug/permalink rename while still published: delete the old vectors before upserting new ones.
5918		- // Without this, md5(old_url) vectors (base + chunks) would be orphaned under the stale URL.
5919		- if ($post->post_status === 'publish' && !empty($previous_url)) {
5920		- $current_url = get_permalink($post_id);
5921		- if ($current_url && $current_url !== $previous_url) {
5922		- MxChat_Utils::delete_chunks_for_url($previous_url, 'default');
5923		- }
5924		- }
5925		-
5926		- // Store the current status for next time (if this is an update)
5927		- if ($update) {
5928		- set_transient($previous_status_key, $post->post_status, DAY_IN_SECONDS);
5929		-
5930		- // If the post is currently published, also store its URL
5931		- if ($post->post_status === 'publish') {
5932		- $current_url = get_permalink($post_id);
5933		- set_transient($previous_url_key, $current_url, DAY_IN_SECONDS);
5934		- }
5935		- }
5936		-
5937		- // Only process currently published content for adding/updating
5938		- if ($post->post_status === 'publish') {
5939		- // Get the source URL
5940		- $source_url = get_permalink($post_id);
5941	3077
5942		- // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
5943		- $title = get_the_title($post_id);
5944		- $content = get_post_field('post_content', $post_id);
5945		- $excerpt = get_post_field('post_excerpt', $post_id);
5946		-
5947		- // Remove shortcode tags but preserve content inside them
5948		- $content = $this->strip_shortcode_tags_preserve_content($content);
5949		- $excerpt = $this->strip_shortcode_tags_preserve_content($excerpt);
5950		-
5951		- // Strip tags but preserve structure (don't use 'the_content' filter as it may re-add shortcodes)
5952		- $content = wp_strip_all_tags($content);
5953		-
5954		- // Combine title, short description (if exists), and content
5955		- $final_content = $title . "\n\n";
5956		-
5957		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
5958		- if (!empty($excerpt)) {
5959		- $final_content .= "Short Description: " . wp_strip_all_tags($excerpt) . "\n\n";
5960		- }
5961		-
5962		- $final_content .= $content;
5963		-
5964		- // For WooCommerce products, include pricing and product details
5965		- if ($post_type === 'product' && class_exists('WooCommerce')) {
5966		- $product = wc_get_product($post_id);
5967		-
5968		- if ($product) {
5969		- // Get pricing information
5970		- $regular_price = $product->get_regular_price();
5971		- $sale_price = $product->get_sale_price();
5972		- $price = $product->get_price();
5973		- $sku = $product->get_sku();
5974		-
5975		- // Get currency symbol
5976		- $currency_symbol = get_woocommerce_currency_symbol();
5977		-
5978		- // Add pricing information
5979		- $final_content .= "\n";
5980		- if (!empty($regular_price)) {
5981		- $final_content .= "Price: " . $currency_symbol . $regular_price . "\n";
5982		- } elseif (!empty($price)) {
5983		- $final_content .= "Price: " . $currency_symbol . $price . "\n";
5984		- }
5985		-
5986		- if (!empty($sale_price) && $sale_price !== $regular_price) {
5987		- $final_content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
5988		- }
5989		-
5990		- // Handle variable products - show price range
5991		- if ($product->is_type('variable')) {
5992		- $min_price = $product->get_variation_price('min');
5993		- $max_price = $product->get_variation_price('max');
5994		- if ($min_price !== $max_price) {
5995		- $final_content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
5996		- }
5997		- }
5998		-
5999		- if (!empty($sku)) {
6000		- $final_content .= "SKU: " . $sku . "\n";
6001		- }
6002		-
6003		- // Get product categories
6004		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
6005		- if (!empty($categories) && !is_wp_error($categories)) {
6006		- $final_content .= "Categories: " . implode(', ', $categories) . "\n";
6007		- }
	3078	+ // Get job type terms
	3079	+ $job_types = get_the_terms($post_id, 'job_listing_type');
	3080	+ if (!empty($job_types) && !is_wp_error($job_types)) {
	3081	+ $types = array();
	3082	+ foreach ($job_types as $type) {
	3083	+ $types[] = $type->name;
6008	3084	}
	3085	+ $final_content .= "\n\nJob Type: " . implode(', ', $types);
6009	3086	}
6010		-
6011		- // For custom post types like job_listing, include additional fields
6012		- if ($post_type === 'job_listing') {
6013		- // Add job-specific meta if available
6014		- $job_location = get_post_meta($post_id, '_job_location', true);
6015		- if (!empty($job_location)) {
6016		- $final_content .= "\n\nLocation: " . $job_location;
6017		- }
6018		-
6019		- // Get job type terms
6020		- $job_types = get_the_terms($post_id, 'job_listing_type');
6021		- if (!empty($job_types) && !is_wp_error($job_types)) {
6022		- $types = array();
6023		- foreach ($job_types as $type) {
6024		- $types[] = $type->name;
6025		- }
6026		- $final_content .= "\n\nJob Type: " . implode(', ', $types);
6027		- }
6028		-
6029		- // Get company name if available
6030		- $company_name = get_post_meta($post_id, '_company_name', true);
6031		- if (!empty($company_name)) {
6032		- $final_content .= "\n\nCompany: " . $company_name;
6033		- }
6034		- }
6035		-
6036		- // ADD ACF FIELDS SUPPORT (matches ajax_mxchat_process_selected_content behavior)
6037		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
6038		- if (!empty($acf_fields)) {
6039		- $acf_content_parts = array();
6040		- $pdf_attachment_ids = array();
6041		-
6042		- foreach ($acf_fields as $field_name => $field_value) {
6043		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
6044		- if (!empty($formatted_value)) {
6045		- // Convert field name to readable label
6046		- $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name));
6047		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
6048		- }
6049		-
6050		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
6051		- }
6052		-
6053		- if (!empty($acf_content_parts)) {
6054		- $final_content .= "\n\n" . implode("\n", $acf_content_parts);
6055		- }
6056		-
6057		- // Gate the auto-sync PDF-extraction loop behind an opt-in option.
6058		- // Mirrors the per-batch checkbox the manual content selector has; the
6059		- // 25 MB size cap lives in the shared extractor so it applies in both
6060		- // paths regardless. Default OFF — re-parsing every ACF PDF on every
6061		- // editor save is expensive and most sites don't want it.
6062		- $autosync_extract_acf_pdfs = get_option('mxchat_auto_sync_acf_pdfs', '0') === '1';
6063		- if ($autosync_extract_acf_pdfs && !empty($pdf_attachment_ids)) {
6064		- $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
6065		- $pdf_sections = array();
6066		- foreach ($pdf_attachment_ids as $att_id) {
6067		- $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
6068		- if (!empty($pdf_text)) {
6069		- $pdf_title = get_the_title($att_id);
6070		- $pdf_url = wp_get_attachment_url($att_id);
6071		- $header = 'PDF Attachment';
6072		- if (!empty($pdf_title)) {
6073		- $header .= ': ' . $pdf_title;
6074		- }
6075		- if (!empty($pdf_url)) {
6076		- $header .= ' (' . $pdf_url . ')';
6077		- }
6078		- $pdf_sections[] = $header . "\n" . $pdf_text;
6079		- }
6080		- }
6081		- if (!empty($pdf_sections)) {
6082		- $final_content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
6083		- }
6084		- }
6085		- }
6086		-
6087		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
6088		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
6089		- if (!empty($custom_meta)) {
6090		- $meta_content_parts = array();
6091		-
6092		- foreach ($custom_meta as $meta_key => $meta_value) {
6093		- // Convert meta key to readable label
6094		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
6095		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
6096		- }
6097		-
6098		- if (!empty($meta_content_parts)) {
6099		- $final_content .= "\n\n" . implode("\n", $meta_content_parts);
6100		- }
6101		- }
6102		-
6103		- // Get API key with proper model detection
6104		- $options = get_option('mxchat_options');
6105		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
6106	3087
6107		- if (strpos($selected_model, 'voyage') === 0) {
6108		- $api_key = $options['voyage_api_key'] ?? '';
6109		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
6110		- $api_key = $options['gemini_api_key'] ?? '';
6111		- } else {
6112		- $api_key = $options['api_key'] ?? '';
	3088	+ // Get company name if available
	3089	+ $company_name = get_post_meta($post_id, '_company_name', true);
	3090	+ if (!empty($company_name)) {
	3091	+ $final_content .= "\n\nCompany: " . $company_name;
6113	3092	}
6114		-
6115		- if (empty($api_key)) {
6116		- return;
6117		- }
6118		-
6119		- // Use the centralized utility function for storage
6120		- $result = MxChat_Utils::submit_content_to_db(
6121		- $final_content,
6122		- $source_url,
6123		- $api_key,
6124		- md5($source_url) // Vector ID for Pinecone
6125		- );
6126		-
6127		- // After successful storage, apply role restriction based on tags
6128		- if (!is_wp_error($result)) {
6129		- $this->apply_role_restriction_to_post($post_id, $source_url);
6130		- }
6131	3093	}
6132	3094
6133		- // Clean up the stored previous status if not used above
6134		- if ($previous_status !== 'publish' || $post->post_status === 'publish') {
6135		- delete_transient($previous_status_key);
6136		- delete_transient($previous_url_key);
	3095	+ // Get the source URL
	3096	+ $source_url = get_permalink($post_id);
	3097	+
	3098	+ // Get API key with proper model detection
	3099	+ $options = get_option('mxchat_options');
	3100	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	3101	+
	3102	+ if (strpos($selected_model, 'voyage') === 0) {
	3103	+ $api_key = $options['voyage_api_key'] ?? '';
	3104	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	3105	+ $api_key = $options['gemini_api_key'] ?? '';
	3106	+ } else {
	3107	+ $api_key = $options['api_key'] ?? '';
6137	3108	}
6138		-}
6139		-
6140		-/**
6141		- * Store the post status and URL before update to detect status transitions
6142		- * This runs before the post is actually updated in the database
6143		- */
6144		-public function mxchat_store_pre_update_status($post_id, $data) {
6145		- // Get the current post from database (before update)
6146		- $current_post = get_post($post_id);
6147	3109
6148		- if ($current_post) {
6149		- // Store the current status temporarily
6150		- $status_key = 'mxchat_prev_status_' . $post_id;
6151		- set_transient($status_key, $current_post->post_status, HOUR_IN_SECONDS);
6152		-
6153		- // If the post is currently published, also store its URL
6154		- if ($current_post->post_status === 'publish') {
6155		- $url_key = 'mxchat_prev_url_' . $post_id;
6156		- $current_url = get_permalink($post_id);
6157		- set_transient($url_key, $current_url, HOUR_IN_SECONDS);
6158		- }
	3110	+ if (empty($api_key)) {
	3111	+ //error_log('MxChat Auto-sync: No API key configured for embedding model');
	3112	+ return;
6159	3113	}
	3114	+
	3115	+ // Use the centralized utility function for storage
	3116	+ $result = MxChat_Utils::submit_content_to_db(
	3117	+ $final_content,
	3118	+ $source_url,
	3119	+ $api_key,
	3120	+ md5($source_url) // Vector ID for Pinecone
	3121	+ );
	3122	+
	3123	+ if (is_wp_error($result)) {
	3124	+ //error_log('MxChat Auto-sync failed for post ' . $post_id . ': ' . $result->get_error_message());
	3125	+ }
6160	3126	}
6161	3127
	3128	+
6162	3129	public function mxchat_handle_post_delete($post_id) {
6163	3130	// Get post data before it's deleted
6164	3131	$post = get_post($post_id);
6165	3132
		@@ -6189,51 +3156,85 @@
6189	3156	if (!$should_sync) {
6190	3157	return;
6191	3158	}
6192	3159
6193		- // Resolve the pre-trash URL. wp_trash_post renames the slug with "__trashed" before firing
6194		- // this hook, so get_permalink() here would return the trashed URL and md5() would miss the
6195		- // real vector IDs stored under the original URL.
6196		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
	3160	+ // Get the URL before post is deleted
	3161	+ $source_url = get_permalink($post_id);
6197	3162	if (!$source_url) {
6198		- //error_log('MXChat: Failed to resolve source URL for post ' . $post_id);
	3163	+ //error_log('MXChat: Failed to get permalink for post ' . $post_id);
6199	3164	return;
6200	3165	}
6201	3166
6202		- // Use chunk-aware deletion (handles both chunked and non-chunked content)
6203		- $delete_result = MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	3167	+ // Check if Pinecone is enabled
	3168	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	3169	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6204	3170
6205		- if (is_wp_error($delete_result)) {
6206		- //error_log('MXChat: Chunk-aware deletion failed for URL: ' . $source_url . ' - ' . $delete_result->get_error_message());
	3171	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3172	+ // Delete from Pinecone
	3173	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
	3174	+ } else {
	3175	+ // Delete from WordPress DB
	3176	+ global $wpdb;
	3177	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	3178	+
	3179	+ $result = $wpdb->delete(
	3180	+ $table_name,
	3181	+ array('source_url' => $source_url),
	3182	+ array('%s')
	3183	+ );
	3184	+
	3185	+ if ($result === false) {
	3186	+ //error_log('MXChat: WordPress DB deletion failed for URL: ' . $source_url);
	3187	+ }
6207	3188	}
	3189	+}
6208	3190
6209		- delete_transient('mxchat_prev_url_' . $post_id);
6210		- delete_transient('mxchat_prev_status_' . $post_id);
6211		-}
6212	3191
6213		-/**
6214		- * Resolve the source URL for a post being trashed/deleted.
6215		- *
6216		- * Why: wp_trash_post appends "__trashed" to the slug before the wp_trash_post action fires, so
6217		- * get_permalink() returns a URL whose md5() won't match the vector IDs stored in Pinecone or
6218		- * the source_url rows in the WP DB. Prefer the URL captured by mxchat_store_pre_update_status
6219		- * (runs on pre_post_update, before the rename); fall back to stripping the __trashed suffix.
6220		- */
6221		-private function mxchat_resolve_pre_trash_url($post_id) {
6222		- $previous_url = get_transient('mxchat_prev_url_' . $post_id);
6223		- if (!empty($previous_url)) {
6224		- return $previous_url;
6225		- }
	3192	+ /**
	3193	+ * Deletes data from Pinecone using a source URL
	3194	+ */
	3195	+ public function mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options) {
	3196	+ $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
	3197	+ $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
6226	3198
6227		- $current = get_permalink($post_id);
6228		- if (!$current) {
6229		- return '';
6230		- }
6231		- return preg_replace('#__trashed(/?)$#', '$1', $current);
6232		-}
	3199	+ if (empty($host) || empty($api_key)) {
	3200	+ //error_log('MXChat: Pinecone deletion failed - missing configuration');
	3201	+ return false;
	3202	+ }
6233	3203
	3204	+ $api_endpoint = "https://{$host}/vectors/delete";
	3205	+ $vector_id = md5($source_url);
6234	3206
	3207	+ $request_body = array(
	3208	+ 'ids' => array($vector_id)
	3209	+ );
6235	3210
	3211	+ $response = wp_remote_post($api_endpoint, array(
	3212	+ 'headers' => array(
	3213	+ 'Api-Key' => $api_key,
	3214	+ 'accept' => 'application/json',
	3215	+ 'content-type' => 'application/json'
	3216	+ ),
	3217	+ 'body' => wp_json_encode($request_body),
	3218	+ 'timeout' => 30
	3219	+ ));
	3220	+
	3221	+ if (is_wp_error($response)) {
	3222	+ //error_log('MXChat: Pinecone deletion error - ' . $response->get_error_message());
	3223	+ return false;
	3224	+ }
	3225	+
	3226	+ $response_code = wp_remote_retrieve_response_code($response);
	3227	+ if ($response_code !== 200) {
	3228	+ //error_log('MXChat: Pinecone deletion failed with status ' . $response_code);
	3229	+ return false;
	3230	+ }
	3231	+
	3232	+ return true;
	3233	+ }
	3234	+
	3235	+
	3236	+
6236	3237	public function mxchat_handle_product_change($post_id, $post, $update) {
6237	3238	if ($post->post_type !== 'product') {
6238	3239	return;
6239	3240	}
		@@ -6255,12 +3256,11 @@
6255	3256	if (!isset($this->options['enable_woocommerce_integration']) ||
6256	3257	!in_array($this->options['enable_woocommerce_integration'], ['1', 'on'])) {
6257	3258	return;
6258	3259	}
6259		-
	3260	+
6260	3261	$source_url = get_permalink($product->get_id());
6261		- $product_id = $product->get_id();
6262		-
	3262	+
6263	3263	// Build product content
6264	3264	$title = $product->get_name();
6265	3265	$description = $product->get_description();
6266	3266	$short_description = $product->get_short_description();
		@@ -6265,92 +3265,35 @@
6265	3265	$description = $product->get_description();
6266	3266	$short_description = $product->get_short_description();
6267	3267	$regular_price = $product->get_regular_price();
6268	3268	$sale_price = $product->get_sale_price();
6269		- $price = $product->get_price();
6270	3269	$sku = $product->get_sku();
6271		-
6272		- // Get currency symbol
6273		- $currency_symbol = get_woocommerce_currency_symbol();
6274		-
	3270	+
6275	3271	// Format content consistently
6276	3272	$content = $title . "\n\n";
6277		-
6278		- if (!empty($short_description)) {
6279		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
6280		- }
6281		-
	3273	+
6282	3274	if (!empty($description)) {
6283	3275	$content .= wp_strip_all_tags($description) . "\n\n";
6284	3276	}
6285		-
6286		- // Add pricing information
6287		- if (!empty($regular_price)) {
6288		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
6289		- } elseif (!empty($price)) {
6290		- $content .= "Price: " . $currency_symbol . $price . "\n";
	3277	+
	3278	+ if (!empty($short_description)) {
	3279	+ $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
6291	3280	}
6292		-
6293		- if (!empty($sale_price) && $sale_price !== $regular_price) {
6294		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
	3281	+
	3282	+ $content .= "Price: $" . $regular_price . "\n";
	3283	+
	3284	+ if (!empty($sale_price)) {
	3285	+ $content .= "Sale Price: $" . $sale_price . "\n";
6295	3286	}
6296		-
6297		- // Handle variable products - show price range
6298		- if ($product->is_type('variable')) {
6299		- $min_price = $product->get_variation_price('min');
6300		- $max_price = $product->get_variation_price('max');
6301		- if ($min_price !== $max_price) {
6302		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
6303		- }
6304		- }
6305		-
	3287	+
6306	3288	if (!empty($sku)) {
6307	3289	$content .= "SKU: " . $sku . "\n";
6308	3290	}
6309		-
6310		- // Get product categories
6311		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
6312		- if (!empty($categories) && !is_wp_error($categories)) {
6313		- $content .= "Categories: " . implode(', ', $categories) . "\n";
6314		- }
6315		-
6316		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
6317		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
6318		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
6319		- foreach ($custom_tabs as $tab) {
6320		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
6321		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
6322		-
6323		- if (!empty($tab_title) && !empty($tab_content)) {
6324		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
6325		- }
6326		- }
6327		- }
6328		-
6329		- // Also check for reusable/saved tabs applied to this product
6330		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
6331		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
6332		- // Get the saved tabs option
6333		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
6334		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
6335		- foreach ($applied_saved_tabs as $saved_tab_id) {
6336		- if (isset($saved_tabs[$saved_tab_id])) {
6337		- $tab = $saved_tabs[$saved_tab_id];
6338		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
6339		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
6340		-
6341		- if (!empty($tab_title) && !empty($tab_content)) {
6342		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
6343		- }
6344		- }
6345		- }
6346		- }
6347		- }
6348		-
	3291	+
6349	3292	// Get API key with proper model detection
6350	3293	$options = get_option('mxchat_options');
6351	3294	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
6352		-
	3295	+
6353	3296	if (strpos($selected_model, 'voyage') === 0) {
6354	3297	$api_key = $options['voyage_api_key'] ?? '';
6355	3298	} elseif (strpos($selected_model, 'gemini-embedding') === 0) {
6356	3299	$api_key = $options['gemini_api_key'] ?? '';
		@@ -6356,14 +3299,14 @@
6356	3299	$api_key = $options['gemini_api_key'] ?? '';
6357	3300	} else {
6358	3301	$api_key = $options['api_key'] ?? '';
6359	3302	}
6360		-
	3303	+
6361	3304	if (empty($api_key)) {
6362	3305	//error_log('MxChat Auto-sync: No API key configured for embedding model');
6363	3306	return;
6364	3307	}
6365		-
	3308	+
6366	3309	// Use the centralized utility function for storage
6367	3310	$result = MxChat_Utils::submit_content_to_db(
6368	3311	$content,
6369	3312	$source_url,
		@@ -6369,16 +3312,11 @@
6369	3312	$source_url,
6370	3313	$api_key,
6371	3314	md5($source_url) // Vector ID for Pinecone
6372	3315	);
6373		-
6374		- // After successful storage, apply role restriction based on tags
6375		- if (!is_wp_error($result)) {
6376		- $this->apply_role_restriction_to_post($product_id, $source_url);
6377		- }
6378		-
	3316	+
6379	3317	if (is_wp_error($result)) {
6380		- //error_log('MxChat WooCommerce sync failed for product ' . $product_id . ': ' . $result->get_error_message());
	3318	+ //error_log('MxChat WooCommerce sync failed for product ' . $product->get_id() . ': ' . $result->get_error_message());
6381	3319	}
6382	3320	}
6383	3321
6384	3322	public function mxchat_handle_product_delete($post_id) {
		@@ -6385,18 +3323,28 @@
6385	3323	if (get_post_type($post_id) !== 'product') {
6386	3324	return;
6387	3325	}
6388	3326
6389		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
6390		- if (!$source_url) {
6391		- return;
6392		- }
	3327	+ $source_url = get_permalink($post_id);
6393	3328
6394		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
6395		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	3329	+ // Check if Pinecone is enabled
	3330	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	3331	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6396	3332
6397		- delete_transient('mxchat_prev_url_' . $post_id);
6398		- delete_transient('mxchat_prev_status_' . $post_id);
	3333	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3334	+ // Delete from Pinecone
	3335	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
	3336	+ } else {
	3337	+ // Delete from WordPress DB
	3338	+ global $wpdb;
	3339	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	3340	+
	3341	+ $wpdb->delete(
	3342	+ $table_name,
	3343	+ array('source_url' => $source_url),
	3344	+ array('%s')
	3345	+ );
	3346	+ }
6399	3347	}
6400	3348
6401	3349	/**
6402	3350	* Handle individual Pinecone content deletion
		@@ -6444,9 +3392,21 @@
6444	3392	$pinecone_options['mxchat_pinecone_host']
6445	3393	);
6446	3394
6447	3395	if ($result['success']) {
6448		- // No cache clearing needed since we removed caching
	3396	+ // Remove from ALL caches
	3397	+ $pinecone_manager->mxchat_remove_from_pinecone_vector_cache($vector_id);
	3398	+ $pinecone_manager->mxchat_remove_from_processed_content_caches($vector_id);
	3399	+
	3400	+ // CLEAR ALL RELEVANT CACHES
	3401	+ delete_transient('mxchat_pinecone_recent_1k_cache');
	3402	+ delete_option('mxchat_pinecone_vector_ids_cache');
	3403	+ delete_option('mxchat_pinecone_processed_cache');
	3404	+ delete_option('mxchat_processed_content_cache');
	3405	+
	3406	+ // Also force refresh for next page load
	3407	+ $pinecone_manager->mxchat_refresh_after_new_content($pinecone_options);
	3408	+
6449	3409	set_transient('mxchat_admin_notice_success',
6450	3410	esc_html__('Entry deleted successfully from Pinecone.', 'mxchat'),
6451	3411	30
6452	3412	);
		@@ -6459,11 +3419,9 @@
6459	3419
6460	3420	wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
6461	3421	exit;
6462	3422	}
6463		-/**
6464		- * Handle individual Pinecone content deletion via AJAX
6465		- */
	3423	+
6466	3424	public function ajax_mxchat_delete_pinecone_prompt() {
6467	3425	// Verify nonce and permissions
6468	3426	if (!check_ajax_referer('mxchat_delete_pinecone_prompt_nonce', 'nonce', false)) {
6469	3427	wp_send_json_error('Invalid nonce');
		@@ -6475,9 +3433,8 @@
6475	3433	exit;
6476	3434	}
6477	3435
6478	3436	$vector_id = isset($_POST['vector_id']) ? sanitize_text_field($_POST['vector_id']) : '';
6479		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6480	3437
6481	3438	if (empty($vector_id)) {
6482	3439	wp_send_json_error('Missing vector ID');
6483	3440	exit;
		@@ -6482,20 +3439,19 @@
6482	3439	wp_send_json_error('Missing vector ID');
6483	3440	exit;
6484	3441	}
6485	3442
6486		- // Get bot-specific Pinecone settings
6487		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6488		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6489		-
	3443	+ // Get Pinecone settings
	3444	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6490	3445	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6491	3446
6492	3447	if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
6493		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
	3448	+ wp_send_json_error('Pinecone is not properly configured');
6494	3449	exit;
6495	3450	}
6496	3451
6497		- // Delete from the correct Pinecone index
	3452	+ // Delete from Pinecone
	3453	+ $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6498	3454	$result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
6499	3455	$vector_id,
6500	3456	$pinecone_options['mxchat_pinecone_api_key'],
6501	3457	$pinecone_options['mxchat_pinecone_host']
		@@ -6501,1007 +3457,32 @@
6501	3457	$pinecone_options['mxchat_pinecone_host']
6502	3458	);
6503	3459
6504	3460	if ($result['success']) {
6505		- // No cache clearing needed since we removed caching
	3461	+ // Remove from ALL caches
	3462	+ $pinecone_manager->mxchat_remove_from_pinecone_vector_cache($vector_id);
	3463	+ $pinecone_manager->mxchat_remove_from_processed_content_caches($vector_id);
	3464	+
	3465	+ // CLEAR ALL RELEVANT CACHES (ADD THESE LINES)
	3466	+ delete_transient('mxchat_pinecone_recent_1k_cache');
	3467	+ delete_option('mxchat_pinecone_vector_ids_cache');
	3468	+ delete_option('mxchat_pinecone_processed_cache');
	3469	+ delete_option('mxchat_processed_content_cache');
	3470	+
	3471	+ // Also force refresh for next page load
	3472	+ $pinecone_manager->mxchat_refresh_after_new_content($pinecone_options);
	3473	+
6506	3474	wp_send_json_success(array(
6507	3475	'message' => 'Entry deleted successfully from Pinecone',
6508		- 'vector_id' => $vector_id,
6509		- 'bot_id' => $bot_id
	3476	+ 'vector_id' => $vector_id
6510	3477	));
6511	3478	} else {
6512		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete from Pinecone: ' . $result['message'], array('vector_id' => $vector_id, 'bot_id' => $bot_id));
6513	3479	wp_send_json_error('Failed to delete from Pinecone: ' . $result['message']);
6514	3480	}
6515	3481
6516	3482	exit;
6517	3483	}
6518		-
6519		-/**
6520		- * Handle deletion of all chunks for a given source URL via AJAX
6521		- * Follows the same pattern as ajax_mxchat_delete_pinecone_prompt
6522		- */
6523		-public function ajax_mxchat_delete_chunks_by_url() {
6524		- // Verify nonce and permissions
6525		- if (!check_ajax_referer('mxchat_delete_chunks_nonce', 'nonce', false)) {
6526		- wp_send_json_error('Invalid nonce');
6527		- exit;
6528		- }
6529		-
6530		- if (!current_user_can('manage_options')) {
6531		- wp_send_json_error('Unauthorized access');
6532		- exit;
6533		- }
6534		-
6535		- $source_url = isset($_POST['source_url']) ? esc_url_raw($_POST['source_url']) : '';
6536		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
6537		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6538		-
6539		- if (empty($source_url)) {
6540		- wp_send_json_error('Missing source URL');
6541		- exit;
6542		- }
6543		-
6544		- // Generate the base vector ID from the source URL (same as how chunks are created)
6545		- $base_vector_id = md5($source_url);
6546		-
6547		- if ($data_source === 'pinecone') {
6548		- // Get bot-specific Pinecone settings (same as working delete function)
6549		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6550		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6551		-
6552		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6553		-
6554		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
6555		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
6556		- exit;
6557		- }
6558		-
6559		- $api_key = $pinecone_options['mxchat_pinecone_api_key'];
6560		- $host = $pinecone_options['mxchat_pinecone_host'];
6561		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
6562		-
6563		- // Collect all vector IDs to delete
6564		- $vectors_to_delete = array();
6565		-
6566		- // Add the original single-vector ID (for non-chunked content)
6567		- $vectors_to_delete[] = $base_vector_id;
6568		-
6569		- // Use Pinecone list API to find all chunk vectors with this prefix
6570		- // NOTE: Pinecone List API is a GET request with query parameters, not POST
6571		- $prefix = $base_vector_id . '_chunk_';
6572		-
6573		- $query_params = array(
6574		- 'prefix' => $prefix,
6575		- 'limit' => 100
6576		- );
6577		-
6578		- if (!empty($namespace)) {
6579		- $query_params['namespace'] = $namespace;
6580		- }
6581		-
6582		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
6583		-
6584		- $list_response = wp_remote_get($list_url, array(
6585		- 'headers' => array(
6586		- 'Api-Key' => $api_key,
6587		- 'accept' => 'application/json'
6588		- ),
6589		- 'timeout' => 30
6590		- ));
6591		-
6592		- if (!is_wp_error($list_response)) {
6593		- $list_body_response = wp_remote_retrieve_body($list_response);
6594		- $list_data = json_decode($list_body_response, true);
6595		- if (!empty($list_data['vectors'])) {
6596		- foreach ($list_data['vectors'] as $vector) {
6597		- if (isset($vector['id'])) {
6598		- $vectors_to_delete[] = $vector['id'];
6599		- }
6600		- }
6601		- }
6602		- }
6603		-
6604		- if (empty($vectors_to_delete)) {
6605		- wp_send_json_success(array(
6606		- 'message' => 'No vectors found to delete',
6607		- 'source_url' => $source_url
6608		- ));
6609		- exit;
6610		- }
6611		-
6612		- // Delete all vectors using the same endpoint as the working function
6613		- $delete_url = "https://{$host}/vectors/delete";
6614		-
6615		- $delete_body = array(
6616		- 'ids' => $vectors_to_delete
6617		- );
6618		-
6619		- if (!empty($namespace)) {
6620		- $delete_body['namespace'] = $namespace;
6621		- }
6622		-
6623		- $delete_response = wp_remote_post($delete_url, array(
6624		- 'headers' => array(
6625		- 'Api-Key' => $api_key,
6626		- 'accept' => 'application/json',
6627		- 'content-type' => 'application/json'
6628		- ),
6629		- 'body' => wp_json_encode($delete_body),
6630		- 'timeout' => 30
6631		- ));
6632		-
6633		- if (is_wp_error($delete_response)) {
6634		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete chunks from Pinecone: ' . $delete_response->get_error_message(), array('source_url' => $source_url));
6635		- wp_send_json_error('Failed to delete from Pinecone: ' . $delete_response->get_error_message());
6636		- exit;
6637		- }
6638		-
6639		- $response_code = wp_remote_retrieve_response_code($delete_response);
6640		-
6641		- if ($response_code !== 200) {
6642		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Pinecone API error (HTTP ' . $response_code . ')', array('source_url' => $source_url));
6643		- wp_send_json_error('Pinecone API error (HTTP ' . $response_code . ')');
6644		- exit;
6645		- }
6646		-
6647		- wp_send_json_success(array(
6648		- 'message' => 'All chunks deleted successfully from Pinecone',
6649		- 'source_url' => $source_url,
6650		- 'deleted_count' => count($vectors_to_delete)
6651		- ));
6652		-
6653		- } else {
6654		- // WordPress database deletion
6655		- global $wpdb;
6656		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6657		-
6658		- $result = $wpdb->delete(
6659		- $table_name,
6660		- array('source_url' => $source_url),
6661		- array('%s')
6662		- );
6663		-
6664		- if ($result === false) {
6665		- MxChat_Admin::mxchat_log_debug('database_error', 'Failed to delete from database: ' . $wpdb->last_error, array('source_url' => $source_url));
6666		- wp_send_json_error('Failed to delete from database: ' . $wpdb->last_error);
6667		- exit;
6668		- }
6669		-
6670		- wp_send_json_success(array(
6671		- 'message' => 'All chunks deleted successfully from database',
6672		- 'source_url' => $source_url,
6673		- 'deleted_count' => $result
6674		- ));
6675		- }
6676		-
6677		- exit;
6678		-}
6679		-
6680		-/**
6681		- * Handle individual WordPress database content deletion via AJAX
6682		- * Mirrors the Pinecone delete handler but for WordPress database entries
6683		- */
6684		-public function ajax_mxchat_delete_wordpress_prompt() {
6685		- // Verify nonce and permissions
6686		- if (!check_ajax_referer('mxchat_delete_wordpress_prompt_nonce', 'nonce', false)) {
6687		- wp_send_json_error('Invalid nonce');
6688		- exit;
6689		- }
6690		-
6691		- if (!current_user_can('manage_options')) {
6692		- wp_send_json_error('Unauthorized access');
6693		- exit;
6694		- }
6695		-
6696		- $entry_id = isset($_POST['entry_id']) ? intval($_POST['entry_id']) : 0;
6697		-
6698		- if (empty($entry_id)) {
6699		- wp_send_json_error('Missing entry ID');
6700		- exit;
6701		- }
6702		-
6703		- global $wpdb;
6704		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6705		-
6706		- // Clear cache for this entry
6707		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
6708		-
6709		- // Delete from database
6710		- $result = $wpdb->delete(
6711		- $table_name,
6712		- array('id' => $entry_id),
6713		- array('%d')
6714		- );
6715		-
6716		- if ($result !== false) {
6717		- wp_send_json_success(array(
6718		- 'message' => 'Entry deleted successfully',
6719		- 'entry_id' => $entry_id
6720		- ));
6721		- } else {
6722		- wp_send_json_error('Failed to delete entry from database');
6723		- }
6724		-
6725		- exit;
6726		-}
6727		-
6728		-/**
6729		- * Handle bulk deletion of knowledge entries via AJAX
6730		- * Supports both Pinecone and WordPress database entries
6731		- */
6732		-public function ajax_mxchat_bulk_delete_knowledge() {
6733		- // Verify nonce and permissions
6734		- if (!check_ajax_referer('mxchat_bulk_delete_knowledge_nonce', 'nonce', false)) {
6735		- wp_send_json_error('Invalid nonce');
6736		- exit;
6737		- }
6738		-
6739		- if (!current_user_can('manage_options')) {
6740		- wp_send_json_error('Unauthorized access');
6741		- exit;
6742		- }
6743		-
6744		- $entries = isset($_POST['entries']) ? $_POST['entries'] : array();
6745		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6746		-
6747		- if (empty($entries) || !is_array($entries)) {
6748		- wp_send_json_error('No entries provided');
6749		- exit;
6750		- }
6751		-
6752		- // Extend execution time — bulk Pinecone operations can take a while
6753		- if (function_exists('set_time_limit')) {
6754		- set_time_limit(120);
6755		- }
6756		-
6757		- $success_ids = array();
6758		- $failed_ids = array();
6759		- $errors = array();
6760		-
6761		- global $wpdb;
6762		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6763		-
6764		- // Get Pinecone manager for Pinecone deletions
6765		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6766		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6767		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6768		-
6769		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
6770		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
6771		-
6772		- // =============================================
6773		- // PHASE 1: Collect all Pinecone vector IDs
6774		- // and separate WordPress entries
6775		- // =============================================
6776		- $pinecone_entry_ids = array(); // entry IDs that are pinecone-sourced
6777		- $wordpress_entries = array(); // entries for WordPress DB deletion
6778		- $all_vector_ids = array(); // all pinecone vector IDs to delete in one batch
6779		-
6780		- foreach ($entries as $entry) {
6781		- $entry_id = sanitize_text_field($entry['id'] ?? '');
6782		- $source = sanitize_text_field($entry['source'] ?? 'wordpress');
6783		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
6784		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
6785		-
6786		- if (empty($entry_id)) {
6787		- continue;
6788		- }
6789		-
6790		- if ($source === 'pinecone') {
6791		- if (!$use_pinecone || empty($api_key)) {
6792		- $failed_ids[] = $entry_id;
6793		- $errors[] = "Pinecone not configured for entry: $entry_id";
6794		- continue;
6795		- }
6796		-
6797		- $pinecone_entry_ids[] = $entry_id;
6798		-
6799		- if ($is_group && !empty($source_url)) {
6800		- // Grouped/chunked entry: collect base ID + chunk IDs via List API
6801		- $base_vector_id = md5($source_url);
6802		- $all_vector_ids[] = $base_vector_id;
6803		-
6804		- $list_url = 'https://' . $host . '/vectors/list?prefix=' . urlencode($base_vector_id . '_chunk_') . '&limit=100';
6805		- $list_response = wp_remote_get($list_url, array(
6806		- 'headers' => array(
6807		- 'Api-Key' => $api_key,
6808		- 'accept' => 'application/json'
6809		- ),
6810		- 'timeout' => 30
6811		- ));
6812		-
6813		- if (!is_wp_error($list_response)) {
6814		- $list_body = json_decode(wp_remote_retrieve_body($list_response), true);
6815		- if (!empty($list_body['vectors']) && is_array($list_body['vectors'])) {
6816		- foreach ($list_body['vectors'] as $vector) {
6817		- if (isset($vector['id'])) {
6818		- $all_vector_ids[] = $vector['id'];
6819		- }
6820		- }
6821		- }
6822		- }
6823		- } else {
6824		- // Single entry: the entry_id IS the vector ID
6825		- $all_vector_ids[] = $entry_id;
6826		- }
6827		- } else {
6828		- $wordpress_entries[] = $entry;
6829		- }
6830		- }
6831		-
6832		- // =============================================
6833		- // PHASE 2: Single batch delete to Pinecone
6834		- // =============================================
6835		- if (!empty($all_vector_ids)) {
6836		- $all_vector_ids = array_values(array_unique($all_vector_ids));
6837		- $pinecone_success = true;
6838		- $batches = array_chunk($all_vector_ids, 100);
6839		-
6840		- foreach ($batches as $batch) {
6841		- $delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
6842		- 'headers' => array(
6843		- 'Api-Key' => $api_key,
6844		- 'accept' => 'application/json',
6845		- 'content-type' => 'application/json'
6846		- ),
6847		- 'body' => wp_json_encode(array('ids' => $batch)),
6848		- 'timeout' => 60
6849		- ));
6850		-
6851		- if (is_wp_error($delete_response)) {
6852		- $pinecone_success = false;
6853		- $errors[] = 'Pinecone batch deletion failed: ' . $delete_response->get_error_message();
6854		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed: ' . $delete_response->get_error_message());
6855		- } else {
6856		- $response_code = wp_remote_retrieve_response_code($delete_response);
6857		- if ($response_code !== 200) {
6858		- $pinecone_success = false;
6859		- $response_body = wp_remote_retrieve_body($delete_response);
6860		- $errors[] = "Pinecone API error (HTTP $response_code)";
6861		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed (HTTP ' . $response_code . ')', array('response' => substr($response_body, 0, 200)));
6862		- }
6863		- }
6864		- }
6865		-
6866		- // Mark all pinecone entries based on batch result
6867		- foreach ($pinecone_entry_ids as $eid) {
6868		- if ($pinecone_success) {
6869		- $success_ids[] = $eid;
6870		- } else {
6871		- $failed_ids[] = $eid;
6872		- }
6873		- }
6874		- }
6875		-
6876		- // =============================================
6877		- // PHASE 3: WordPress database deletions
6878		- // =============================================
6879		- foreach ($wordpress_entries as $entry) {
6880		- $entry_id = sanitize_text_field($entry['id'] ?? '');
6881		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
6882		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
6883		-
6884		- if (empty($entry_id)) {
6885		- continue;
6886		- }
6887		-
6888		- try {
6889		- if ($is_group && !empty($source_url)) {
6890		- $result = $wpdb->delete(
6891		- $table_name,
6892		- array('source_url' => $source_url),
6893		- array('%s')
6894		- );
6895		- } else {
6896		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
6897		- $result = $wpdb->delete(
6898		- $table_name,
6899		- array('id' => intval($entry_id)),
6900		- array('%d')
6901		- );
6902		- }
6903		-
6904		- if ($result !== false) {
6905		- $success_ids[] = $entry_id;
6906		- } else {
6907		- $failed_ids[] = $entry_id;
6908		- $errors[] = "Database error for entry: $entry_id";
6909		- }
6910		- } catch (Exception $e) {
6911		- $failed_ids[] = $entry_id;
6912		- $errors[] = $e->getMessage();
6913		- }
6914		- }
6915		-
6916		- wp_send_json_success(array(
6917		- 'success_ids' => $success_ids,
6918		- 'failed_ids' => $failed_ids,
6919		- 'errors' => $errors,
6920		- 'total_processed' => count($success_ids) + count($failed_ids)
6921		- ));
6922		-
6923		- exit;
6924		-}
6925		-
6926		-/**
6927		- * Get hierarchical roles for dropdown
6928		- */
6929		-public function mxchat_get_role_options() {
6930		- return array(
6931		- 'public' => __('Public (Everyone)', 'mxchat'),
6932		- 'logged_in' => __('Logged In Users', 'mxchat'),
6933		- 'subscriber' => __('Subscribers & Above', 'mxchat'),
6934		- 'contributor' => __('Contributors & Above', 'mxchat'),
6935		- 'author' => __('Authors & Above', 'mxchat'),
6936		- 'editor' => __('Editors & Above', 'mxchat'),
6937		- 'administrator' => __('Administrators Only', 'mxchat')
6938		- );
6939		-}
6940		-
6941		-/**
6942		- * Check if user has access to content based on role restriction
6943		- */
6944		-public function mxchat_user_has_content_access($role_restriction) {
6945		- // Public content is always accessible
6946		- if ($role_restriction === 'public' || empty($role_restriction)) {
6947		- return true;
6948		- }
6949	3484
6950		- // Check if user is logged in for logged_in restriction
6951		- if ($role_restriction === 'logged_in') {
6952		- return is_user_logged_in();
6953		- }
6954		-
6955		- // If not logged in, no access to role-restricted content
6956		- if (!is_user_logged_in()) {
6957		- return false;
6958		- }
6959		-
6960		- $user = wp_get_current_user();
6961		- $user_roles = $user->roles;
6962		-
6963		- if (empty($user_roles)) {
6964		- return false;
6965		- }
6966		-
6967		- // Define role hierarchy (higher number = higher access)
6968		- $hierarchy = array(
6969		- 'subscriber' => 1,
6970		- 'contributor' => 2,
6971		- 'author' => 3,
6972		- 'editor' => 4,
6973		- 'administrator' => 5
6974		- );
6975		-
6976		- // Get required level
6977		- $required_level = isset($hierarchy[$role_restriction]) ? $hierarchy[$role_restriction] : 0;
6978		-
6979		- // Check if user has required level or higher
6980		- foreach ($user_roles as $user_role) {
6981		- $user_level = isset($hierarchy[$user_role]) ? $hierarchy[$user_role] : 0;
6982		- if ($user_level >= $required_level) {
6983		- return true;
6984		- }
6985		- }
6986		-
6987		- return false;
6988		-}
6989		-
6990		-/**
6991		- * Handle role restriction updates via AJAX
6992		- * Removed cache clearing call since we removed caching
6993		- */
6994		-public function ajax_mxchat_update_role_restriction() {
6995		- // Verify nonce and permissions
6996		- if (!check_ajax_referer('mxchat_update_role_nonce', 'nonce', false)) {
6997		- wp_send_json_error('Invalid nonce');
6998		- exit;
6999		- }
7000		-
7001		- if (!current_user_can('manage_options')) {
7002		- wp_send_json_error('Unauthorized access');
7003		- exit;
7004		- }
7005		-
7006		- $entry_id = isset($_POST['entry_id']) ? sanitize_text_field($_POST['entry_id']) : '';
7007		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
7008		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
7009		-
7010		- if (empty($entry_id)) {
7011		- wp_send_json_error('Invalid entry ID');
7012		- exit;
7013		- }
7014		-
7015		- // Get knowledge manager instance to validate role restriction
7016		- $knowledge_manager = MxChat_Knowledge_Manager::get_instance();
7017		- $valid_roles = array_keys($knowledge_manager->mxchat_get_role_options());
7018		- if (!in_array($role_restriction, $valid_roles)) {
7019		- wp_send_json_error('Invalid role restriction');
7020		- exit;
7021		- }
7022		-
7023		- global $wpdb;
7024		-
7025		- if ($data_source === 'pinecone') {
7026		- // Handle Pinecone role restriction (stored separately in WordPress table)
7027		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7028		-
7029		- // Use REPLACE to insert or update the role restriction
7030		- $result = $wpdb->replace(
7031		- $roles_table,
7032		- array(
7033		- 'vector_id' => $entry_id,
7034		- 'role_restriction' => $role_restriction,
7035		- 'updated_at' => current_time('mysql')
7036		- ),
7037		- array('%s', '%s', '%s')
7038		- );
7039		-
7040		- // No cache clearing needed since we removed caching
7041		-
7042		- } else {
7043		- // Handle WordPress database role restriction (existing functionality)
7044		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7045		-
7046		- $result = $wpdb->update(
7047		- $table_name,
7048		- array('role_restriction' => $role_restriction),
7049		- array('id' => absint($entry_id)),
7050		- array('%s'),
7051		- array('%d')
7052		- );
7053		- }
7054		-
7055		- if ($result === false) {
7056		- wp_send_json_error('Database update failed: ' . $wpdb->last_error);
7057		- exit;
7058		- }
7059		-
7060		- wp_send_json_success(array(
7061		- 'message' => 'Role restriction updated successfully',
7062		- 'role_restriction' => $role_restriction,
7063		- 'data_source' => $data_source,
7064		- 'entry_id' => $entry_id
7065		- ));
7066		- exit;
7067		-}
7068		-
7069		-// ========================================
7070		-// ROLE-BASED CONTENT RESTRICTIONS - NEW FUNCTIONS
7071		-// Add these to your MxChat_Knowledge_Manager class
7072		-// ========================================
7073		-
7074		-/**
7075		- * Initialize role-based content hooks
7076		- * Add this call to your __construct() or mxchat_init_hooks() method
7077		- */
7078		-private function mxchat_init_role_hooks() {
7079		- // AJAX handlers for tag-role mappings
7080		- add_action('wp_ajax_mxchat_add_tag_role_mapping', array($this, 'ajax_add_tag_role_mapping'));
7081		- add_action('wp_ajax_mxchat_delete_tag_role_mapping', array($this, 'ajax_delete_tag_role_mapping'));
7082		- add_action('wp_ajax_mxchat_get_tag_role_mappings', array($this, 'ajax_get_tag_role_mappings'));
7083		- add_action('wp_ajax_mxchat_bulk_update_tag_roles', array($this, 'ajax_bulk_update_tag_roles'));
7084		-
7085		- // Hook to automatically update role restrictions when tags are added/removed
7086		- add_action('set_object_terms', array($this, 'handle_tag_change'), 10, 6);
7087		-
7088		- // Hook to apply role restrictions on auto-sync
7089		- add_action('mxchat_content_stored', array($this, 'apply_role_restriction_after_storage'), 10, 2);
7090		-}
7091		-
7092		-/**
7093		- * Add tag-role mapping via AJAX
7094		- */
7095		-public function ajax_add_tag_role_mapping() {
7096		- // Verify nonce and permissions
7097		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7098		-
7099		- if (!current_user_can('manage_options')) {
7100		- wp_send_json_error('Unauthorized access');
7101		- exit;
7102		- }
7103		-
7104		- $tag_input = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
7105		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
7106		-
7107		- if (empty($tag_input)) {
7108		- wp_send_json_error('Please enter a tag name or slug');
7109		- exit;
7110		- }
7111		-
7112		- // Validate role restriction
7113		- $valid_roles = array_keys($this->mxchat_get_role_options());
7114		- if (!in_array($role_restriction, $valid_roles)) {
7115		- wp_send_json_error('Invalid role restriction');
7116		- exit;
7117		- }
7118		-
7119		- // Resolve the tag by slug first, then fall back to its display name, so users can
7120		- // enter either "premium-content" or "Premium Content". (plan b8bcf5 — the field is
7121		- // labeled by name but previously validated by slug only, producing the confusing
7122		- // "Tag does not exist in WordPress" error when a real tag's name was typed.)
7123		- $term = get_term_by('slug', $tag_input, 'post_tag');
7124		- if (!$term) {
7125		- $term = get_term_by('name', $tag_input, 'post_tag');
7126		- }
7127		- if (!$term) {
7128		- wp_send_json_error('No tag with that name or slug exists yet. Create it under Posts → Tags first, then enter its name or slug.');
7129		- exit;
7130		- }
7131		-
7132		- // Always key the mapping by the RESOLVED slug — apply_role_restriction_to_post()
7133		- // compares against each post's tag slugs, so the stored key must be a slug,
7134		- // never the raw (possibly display-name) input.
7135		- $tag_slug = $term->slug;
7136		-
7137		- // Get existing mappings
7138		- $mappings = get_option('mxchat_tag_role_mappings', array());
7139		-
7140		- // Check if mapping already exists
7141		- if (isset($mappings[$tag_slug])) {
7142		- wp_send_json_error('Mapping for this tag already exists');
7143		- exit;
7144		- }
7145		-
7146		- // Add new mapping
7147		- $mappings[$tag_slug] = $role_restriction;
7148		- update_option('mxchat_tag_role_mappings', $mappings);
7149		-
7150		- wp_send_json_success(array(
7151		- 'message' => 'Tag-role mapping added successfully',
7152		- 'tag_slug' => $tag_slug,
7153		- 'role_restriction' => $role_restriction
7154		- ));
7155		- exit;
7156		-}
7157		-
7158		-/**
7159		- * Delete tag-role mapping via AJAX
7160		- */
7161		-public function ajax_delete_tag_role_mapping() {
7162		- // Verify nonce and permissions
7163		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7164		-
7165		- if (!current_user_can('manage_options')) {
7166		- wp_send_json_error('Unauthorized access');
7167		- exit;
7168		- }
7169		-
7170		- $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
7171		-
7172		- if (empty($tag_slug)) {
7173		- wp_send_json_error('Tag slug is required');
7174		- exit;
7175		- }
7176		-
7177		- // Get existing mappings
7178		- $mappings = get_option('mxchat_tag_role_mappings', array());
7179		-
7180		- // Check if mapping exists
7181		- if (!isset($mappings[$tag_slug])) {
7182		- wp_send_json_error('Mapping does not exist');
7183		- exit;
7184		- }
7185		-
7186		- // Remove mapping
7187		- unset($mappings[$tag_slug]);
7188		- update_option('mxchat_tag_role_mappings', $mappings);
7189		-
7190		- wp_send_json_success(array(
7191		- 'message' => 'Tag-role mapping deleted successfully',
7192		- 'tag_slug' => $tag_slug
7193		- ));
7194		- exit;
7195		-}
7196		-
7197		-/**
7198		- * Get all tag-role mappings via AJAX
7199		- */
7200		-public function ajax_get_tag_role_mappings() {
7201		- // Verify nonce and permissions
7202		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7203		-
7204		- if (!current_user_can('manage_options')) {
7205		- wp_send_json_error('Unauthorized access');
7206		- exit;
7207		- }
7208		-
7209		- // Get mappings
7210		- $mappings = get_option('mxchat_tag_role_mappings', array());
7211		- $role_options = $this->mxchat_get_role_options();
7212		-
7213		- $formatted_mappings = array();
7214		-
7215		- foreach ($mappings as $tag_slug => $role_restriction) {
7216		- // Get tag object
7217		- $term = get_term_by('slug', $tag_slug, 'post_tag');
7218		-
7219		- // Count posts with this tag
7220		- $post_count = 0;
7221		- if ($term) {
7222		- $post_count = $term->count;
7223		- }
7224		-
7225		- $formatted_mappings[] = array(
7226		- 'tag_slug' => $tag_slug,
7227		- 'role_restriction' => $role_restriction,
7228		- 'role_label' => $role_options[$role_restriction] ?? $role_restriction,
7229		- 'post_count' => $post_count
7230		- );
7231		- }
7232		-
7233		- wp_send_json_success(array(
7234		- 'mappings' => $formatted_mappings
7235		- ));
7236		- exit;
7237		-}
7238		-
7239		-/**
7240		- * Bulk update role restrictions for all existing content with mapped tags
7241		- */
7242		-public function ajax_bulk_update_tag_roles() {
7243		- // Verify nonce and permissions
7244		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7245		-
7246		- if (!current_user_can('manage_options')) {
7247		- wp_send_json_error('Unauthorized access');
7248		- exit;
7249		- }
7250		-
7251		- // Get mappings
7252		- $mappings = get_option('mxchat_tag_role_mappings', array());
7253		-
7254		- if (empty($mappings)) {
7255		- wp_send_json_error('No tag-role mappings found');
7256		- exit;
7257		- }
7258		-
7259		- global $wpdb;
7260		-
7261		- // Check if using Pinecone
7262		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
7263		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7264		-
7265		- $updated_count = 0;
7266		- $details = array();
7267		-
7268		- foreach ($mappings as $tag_slug => $role_restriction) {
7269		- // Get all posts with this tag
7270		- $posts = get_posts(array(
7271		- 'tag' => $tag_slug,
7272		- 'post_type' => 'any',
7273		- 'posts_per_page' => -1,
7274		- 'fields' => 'ids',
7275		- 'post_status' => 'publish'
7276		- ));
7277		-
7278		- if (empty($posts)) {
7279		- continue;
7280		- }
7281		-
7282		- $tag_updated = 0;
7283		-
7284		- foreach ($posts as $post_id) {
7285		- $source_url = get_permalink($post_id);
7286		- if (!$source_url) {
7287		- continue;
7288		- }
7289		-
7290		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
7291		- // Update Pinecone role restriction
7292		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7293		- $vector_id = md5($source_url);
7294		-
7295		- $result = $wpdb->replace(
7296		- $roles_table,
7297		- array(
7298		- 'vector_id' => $vector_id,
7299		- 'role_restriction' => $role_restriction,
7300		- 'updated_at' => current_time('mysql')
7301		- ),
7302		- array('%s', '%s', '%s')
7303		- );
7304		- } else {
7305		- // Update WordPress DB
7306		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7307		-
7308		- $result = $wpdb->update(
7309		- $table_name,
7310		- array('role_restriction' => $role_restriction),
7311		- array('source_url' => $source_url),
7312		- array('%s'),
7313		- array('%s')
7314		- );
7315		- }
7316		-
7317		- if ($result !== false) {
7318		- $tag_updated++;
7319		- $updated_count++;
7320		- }
7321		- }
7322		-
7323		- if ($tag_updated > 0) {
7324		- $details[] = sprintf(
7325		- 'Tag "%s" (%s): %d posts updated',
7326		- $tag_slug,
7327		- $role_restriction,
7328		- $tag_updated
7329		- );
7330		- }
7331		- }
7332		-
7333		- wp_send_json_success(array(
7334		- 'message' => 'Bulk update completed',
7335		- 'updated_count' => $updated_count,
7336		- 'tags_processed' => count($mappings),
7337		- 'details' => $details
7338		- ));
7339		- exit;
7340		-}
7341		-
7342		-/**
7343		- * Handle tag changes on posts (when tags are added or removed)
7344		- */
7345		-public function handle_tag_change($object_id, $terms, $tt_ids, $taxonomy, $append, $old_tt_ids) {
7346		- // Only process post tags
7347		- if ($taxonomy !== 'post_tag') {
7348		- return;
7349		- }
7350		-
7351		- // Get tag-role mappings
7352		- $mappings = get_option('mxchat_tag_role_mappings', array());
7353		-
7354		- if (empty($mappings)) {
7355		- return;
7356		- }
7357		-
7358		- // Get the post's URL
7359		- $source_url = get_permalink($object_id);
7360		- if (!$source_url) {
7361		- return;
7362		- }
7363		-
7364		- // Determine the highest role restriction based on tags
7365		- $highest_role = 'public';
7366		- $role_hierarchy = array(
7367		- 'public' => 0,
7368		- 'logged_in' => 1,
7369		- 'subscriber' => 2,
7370		- 'contributor' => 3,
7371		- 'author' => 4,
7372		- 'editor' => 5,
7373		- 'administrator' => 6
7374		- );
7375		-
7376		- // Get all current tags for the post
7377		- $current_tags = wp_get_post_tags($object_id, array('fields' => 'slugs'));
7378		-
7379		- // Find the highest role restriction among the tags
7380		- foreach ($current_tags as $tag_slug) {
7381		- if (isset($mappings[$tag_slug])) {
7382		- $role = $mappings[$tag_slug];
7383		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
7384		- $highest_role = $role;
7385		- }
7386		- }
7387		- }
7388		-
7389		- // Update the role restriction in the database
7390		- global $wpdb;
7391		-
7392		- // Check if using Pinecone
7393		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
7394		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7395		-
7396		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
7397		- // Update Pinecone role restriction
7398		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7399		- $vector_id = md5($source_url);
7400		-
7401		- $wpdb->replace(
7402		- $roles_table,
7403		- array(
7404		- 'vector_id' => $vector_id,
7405		- 'role_restriction' => $highest_role,
7406		- 'updated_at' => current_time('mysql')
7407		- ),
7408		- array('%s', '%s', '%s')
7409		- );
7410		- } else {
7411		- // Update WordPress DB
7412		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7413		-
7414		- $wpdb->update(
7415		- $table_name,
7416		- array('role_restriction' => $highest_role),
7417		- array('source_url' => $source_url),
7418		- array('%s'),
7419		- array('%s')
7420		- );
7421		- }
7422		-}
7423		-
7424		-/**
7425		- * Apply role restriction after content is stored (for auto-sync)
7426		- */
7427		-public function apply_role_restriction_after_storage($post_id, $source_url) {
7428		- // Get tag-role mappings
7429		- $mappings = get_option('mxchat_tag_role_mappings', array());
7430		-
7431		- if (empty($mappings)) {
7432		- return;
7433		- }
7434		-
7435		- // Get all tags for the post
7436		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
7437		-
7438		- if (empty($post_tags)) {
7439		- return;
7440		- }
7441		-
7442		- // Determine the highest role restriction based on tags
7443		- $highest_role = 'public';
7444		- $role_hierarchy = array(
7445		- 'public' => 0,
7446		- 'logged_in' => 1,
7447		- 'subscriber' => 2,
7448		- 'contributor' => 3,
7449		- 'author' => 4,
7450		- 'editor' => 5,
7451		- 'administrator' => 6
7452		- );
7453		-
7454		- foreach ($post_tags as $tag_slug) {
7455		- if (isset($mappings[$tag_slug])) {
7456		- $role = $mappings[$tag_slug];
7457		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
7458		- $highest_role = $role;
7459		- }
7460		- }
7461		- }
7462		-
7463		- // If no restricted tags found, return (leave as public)
7464		- if ($highest_role === 'public') {
7465		- return;
7466		- }
7467		-
7468		- // Update the role restriction
7469		- global $wpdb;
7470		-
7471		- // Check if using Pinecone
7472		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
7473		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7474		-
7475		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
7476		- // Update Pinecone role restriction
7477		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7478		- $vector_id = md5($source_url);
7479		-
7480		- $wpdb->replace(
7481		- $roles_table,
7482		- array(
7483		- 'vector_id' => $vector_id,
7484		- 'role_restriction' => $highest_role,
7485		- 'updated_at' => current_time('mysql')
7486		- ),
7487		- array('%s', '%s', '%s')
7488		- );
7489		- } else {
7490		- // Update WordPress DB
7491		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7492		-
7493		- $wpdb->update(
7494		- $table_name,
7495		- array('role_restriction' => $highest_role),
7496		- array('source_url' => $source_url),
7497		- array('%s'),
7498		- array('%s')
7499		- );
7500		- }
7501		-}
7502		-
7503		-
7504	3485	// ========================================
7505	3486	// HELPER METHODS
7506	3487	// ========================================
7507	3488
		@@ -7568,1074 +3549,8 @@
7568	3549	*/
7569	3550	private function mxchat_get_pinecone_manager() {
7570	3551	return MxChat_Pinecone_Manager::get_instance();
7571	3552	}
7572		-
7573		-
7574		- // ========================================
7575		-// DATABASE QUEUE TABLE MANAGEMENT
7576		-// ========================================
7577		-
7578		-/**
7579		- * Create queue table on plugin activation
7580		- * Call this from your plugin activation hook
7581		- */
7582		-public function mxchat_create_queue_table() {
7583		- global $wpdb;
7584		-
7585		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7586		- $charset_collate = $wpdb->get_charset_collate();
7587		-
7588		- $sql = "CREATE TABLE IF NOT EXISTS $table_name (
7589		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
7590		- queue_id varchar(64) NOT NULL,
7591		- item_type varchar(20) NOT NULL,
7592		- item_data longtext NOT NULL,
7593		- status varchar(20) NOT NULL DEFAULT 'pending',
7594		- bot_id varchar(50) NOT NULL DEFAULT 'default',
7595		- priority int(11) NOT NULL DEFAULT 0,
7596		- attempts int(11) NOT NULL DEFAULT 0,
7597		- max_attempts int(11) NOT NULL DEFAULT 3,
7598		- error_message text DEFAULT NULL,
7599		- created_at datetime NOT NULL,
7600		- started_at datetime DEFAULT NULL,
7601		- completed_at datetime DEFAULT NULL,
7602		- PRIMARY KEY (id),
7603		- KEY queue_id (queue_id),
7604		- KEY status (status),
7605		- KEY item_type (item_type),
7606		- KEY priority (priority)
7607		- ) $charset_collate;";
7608		-
7609		- require_once(ABSPATH . 'wp-admin/includes/upgrade.php');
7610		- dbDelta($sql);
7611		-
7612		- // Also create a meta table for queue metadata
7613		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7614		-
7615		- $meta_sql = "CREATE TABLE IF NOT EXISTS $meta_table (
7616		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
7617		- queue_id varchar(64) NOT NULL,
7618		- meta_key varchar(255) NOT NULL,
7619		- meta_value longtext,
7620		- PRIMARY KEY (id),
7621		- KEY queue_id (queue_id),
7622		- KEY meta_key (meta_key)
7623		- ) $charset_collate;";
7624		-
7625		- dbDelta($meta_sql);
7626		-}
7627		-
7628		-/**
7629		- * Add items to the processing queue
7630		- *
7631		- * @param string $queue_id Unique identifier for this queue batch
7632		- * @param string $item_type Type of item (url, pdf_page)
7633		- * @param array $items Array of items to queue
7634		- * @param string $bot_id Bot ID for processing
7635		- * @return int Number of items queued
7636		- */
7637		-private function mxchat_add_to_queue($queue_id, $item_type, $items, $bot_id = 'default') {
7638		- global $wpdb;
7639		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7640		-
7641		- $queued_count = 0;
7642		- $priority = 0;
7643		-
7644		- foreach ($items as $item) {
7645		- $result = $wpdb->insert(
7646		- $table_name,
7647		- array(
7648		- 'queue_id' => $queue_id,
7649		- 'item_type' => $item_type,
7650		- 'item_data' => wp_json_encode($item),
7651		- 'status' => 'pending',
7652		- 'bot_id' => $bot_id,
7653		- 'priority' => $priority,
7654		- 'attempts' => 0,
7655		- 'max_attempts' => 3,
7656		- 'created_at' => current_time('mysql')
7657		- ),
7658		- array('%s', '%s', '%s', '%s', '%s', '%d', '%d', '%d', '%s')
7659		- );
7660		-
7661		- if ($result) {
7662		- $queued_count++;
7663		- }
7664		-
7665		- $priority++; // Process in order
7666		- }
7667		-
7668		- return $queued_count;
7669		-}
7670		-
7671		-/**
7672		- * Store queue metadata (total counts, source URL, etc.)
7673		- */
7674		-private function mxchat_set_queue_meta($queue_id, $meta_key, $meta_value) {
7675		- global $wpdb;
7676		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7677		-
7678		- // Check if meta exists
7679		- $existing = $wpdb->get_var($wpdb->prepare(
7680		- "SELECT id FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
7681		- $queue_id,
7682		- $meta_key
7683		- ));
7684		-
7685		- if ($existing) {
7686		- // Update
7687		- $wpdb->update(
7688		- $meta_table,
7689		- array('meta_value' => maybe_serialize($meta_value)),
7690		- array('queue_id' => $queue_id, 'meta_key' => $meta_key),
7691		- array('%s'),
7692		- array('%s', '%s')
7693		- );
7694		- } else {
7695		- // Insert
7696		- $wpdb->insert(
7697		- $meta_table,
7698		- array(
7699		- 'queue_id' => $queue_id,
7700		- 'meta_key' => $meta_key,
7701		- 'meta_value' => maybe_serialize($meta_value)
7702		- ),
7703		- array('%s', '%s', '%s')
7704		- );
7705		- }
7706		-}
7707		-
7708		-/**
7709		- * Get queue metadata
7710		- */
7711		-private function mxchat_get_queue_meta($queue_id, $meta_key) {
7712		- global $wpdb;
7713		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7714		-
7715		- $value = $wpdb->get_var($wpdb->prepare(
7716		- "SELECT meta_value FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
7717		- $queue_id,
7718		- $meta_key
7719		- ));
7720		-
7721		- return maybe_unserialize($value);
7722		-}
7723		-
7724		-// ========================================
7725		-// AJAX QUEUE PROCESSING HANDLERS
7726		-// ========================================
7727		-
7728		-/**
7729		- * AJAX: Get next item from queue to process
7730		- */
7731		-public function ajax_mxchat_get_next_queue_item() {
7732		- // Verify nonce and permissions
7733		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7734		-
7735		- if (!current_user_can('manage_options')) {
7736		- wp_send_json_error('Unauthorized access');
7737		- }
7738		-
7739		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7740		-
7741		- if (empty($queue_id)) {
7742		- wp_send_json_error('Missing queue ID');
7743		- }
7744		-
7745		- global $wpdb;
7746		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7747		-
7748		- // Get next pending item with retry logic for failed items
7749		- $next_item = $wpdb->get_row($wpdb->prepare(
7750		- "SELECT * FROM $table_name
7751		- WHERE queue_id = %s
7752		- AND status IN ('pending', 'failed')
7753		- AND attempts < max_attempts
7754		- ORDER BY priority ASC, id ASC
7755		- LIMIT 1",
7756		- $queue_id
7757		- ));
7758		-
7759		- if (!$next_item) {
7760		- // No more items - queue complete
7761		- wp_send_json_success(array(
7762		- 'complete' => true,
7763		- 'message' => 'Queue processing complete'
7764		- ));
7765		- }
7766		-
7767		- // Mark item as processing
7768		- $wpdb->update(
7769		- $table_name,
7770		- array(
7771		- 'status' => 'processing',
7772		- 'started_at' => current_time('mysql'),
7773		- 'attempts' => $next_item->attempts + 1
7774		- ),
7775		- array('id' => $next_item->id),
7776		- array('%s', '%s', '%d'),
7777		- array('%d')
7778		- );
7779		-
7780		- wp_send_json_success(array(
7781		- 'complete' => false,
7782		- 'item' => array(
7783		- 'id' => $next_item->id,
7784		- 'type' => $next_item->item_type,
7785		- 'data' => json_decode($next_item->item_data, true),
7786		- 'bot_id' => $next_item->bot_id,
7787		- 'attempt' => $next_item->attempts + 1
7788		- )
7789		- ));
7790		-}
7791		-
7792		-/**
7793		- * AJAX: Process a single queue item
7794		- */
7795		-public function ajax_mxchat_process_queue_item() {
7796		- // Verify nonce and permissions
7797		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7798		-
7799		- if (!current_user_can('manage_options')) {
7800		- wp_send_json_error('Unauthorized access');
7801		- }
7802		-
7803		- $item_id = isset($_POST['item_id']) ? absint($_POST['item_id']) : 0;
7804		- $item_type = isset($_POST['item_type']) ? sanitize_text_field($_POST['item_type']) : '';
7805		- $item_data = isset($_POST['item_data']) ? $_POST['item_data'] : array();
7806		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
7807		-
7808		- if (empty($item_id) || empty($item_type)) {
7809		- wp_send_json_error('Missing item data');
7810		- }
7811		-
7812		- global $wpdb;
7813		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7814		-
7815		- // Process based on item type
7816		- try {
7817		- set_time_limit(60); // Give processing 60 seconds
7818		-
7819		- $result = false;
7820		- $error_message = '';
7821		-
7822		- // Read item directly from DB to get queue_id and preserve special chars in item_data
7823		- // (POST round-trip through JS mangles characters like apostrophes in URLs)
7824		- $db_item = $wpdb->get_row($wpdb->prepare(
7825		- "SELECT queue_id, item_data FROM $table_name WHERE id = %d",
7826		- $item_id
7827		- ));
7828		- $item_queue_id = $db_item ? $db_item->queue_id : '';
7829		- if ($db_item && !empty($db_item->item_data)) {
7830		- $db_data = json_decode($db_item->item_data, true);
7831		- if (is_array($db_data)) {
7832		- $item_data = $db_data;
7833		- }
7834		- }
7835		-
7836		- switch ($item_type) {
7837		- case 'url':
7838		- $result = $this->mxchat_process_queue_url($item_data, $bot_id, $item_queue_id);
7839		- break;
7840		-
7841		- case 'pdf_page':
7842		- $result = $this->mxchat_process_queue_pdf_page($item_data, $bot_id);
7843		- break;
7844		-
7845		- default:
7846		- throw new Exception('Unknown item type: ' . $item_type);
7847		- }
7848		-
7849		- if (is_wp_error($result)) {
7850		- $error_code = $result->get_error_code();
7851		- // Content errors (empty page, sanitization) are permanent — retrying won't help
7852		- $permanent_codes = array('empty_page', 'empty_after_sanitization', 'no_api_key', 'page_not_found');
7853		- if (in_array($error_code, $permanent_codes)) {
7854		- // Mark as permanently failed — set attempts = max_attempts so it won't be retried
7855		- $current_item = $wpdb->get_row($wpdb->prepare(
7856		- "SELECT max_attempts FROM $table_name WHERE id = %d", $item_id
7857		- ));
7858		- $wpdb->update(
7859		- $table_name,
7860		- array(
7861		- 'status' => 'failed',
7862		- 'error_message' => $result->get_error_message(),
7863		- 'attempts' => $current_item ? $current_item->max_attempts : 3
7864		- ),
7865		- array('id' => $item_id),
7866		- array('%s', '%s', '%d'),
7867		- array('%d')
7868		- );
7869		- wp_send_json_error(array(
7870		- 'message' => $result->get_error_message(),
7871		- 'permanent_failure' => true,
7872		- 'item_id' => $item_id
7873		- ));
7874		- return;
7875		- }
7876		- throw new Exception($result->get_error_message());
7877		- }
7878		-
7879		- if ($result === false) {
7880		- throw new Exception('Processing returned false - item may be empty or invalid');
7881		- }
7882		-
7883		- // Mark as completed
7884		- $wpdb->update(
7885		- $table_name,
7886		- array(
7887		- 'status' => 'completed',
7888		- 'completed_at' => current_time('mysql'),
7889		- 'error_message' => null
7890		- ),
7891		- array('id' => $item_id),
7892		- array('%s', '%s', '%s'),
7893		- array('%d')
7894		- );
7895		-
7896		- wp_send_json_success(array(
7897		- 'processed' => true,
7898		- 'item_id' => $item_id,
7899		- 'message' => 'Item processed successfully'
7900		- ));
7901		-
7902		- } catch (Exception $e) {
7903		- $error_message = $e->getMessage();
7904		-
7905		- // Get current attempt count
7906		- $item = $wpdb->get_row($wpdb->prepare(
7907		- "SELECT attempts, max_attempts FROM $table_name WHERE id = %d",
7908		- $item_id
7909		- ));
7910		-
7911		- // Check if we've exhausted retries
7912		- if ($item && $item->attempts >= $item->max_attempts) {
7913		- // Permanently failed
7914		- $wpdb->update(
7915		- $table_name,
7916		- array(
7917		- 'status' => 'failed',
7918		- 'error_message' => $error_message
7919		- ),
7920		- array('id' => $item_id),
7921		- array('%s', '%s'),
7922		- array('%d')
7923		- );
7924		-
7925		- wp_send_json_error(array(
7926		- 'message' => 'Item failed after maximum attempts: ' . $error_message,
7927		- 'permanent_failure' => true,
7928		- 'item_id' => $item_id
7929		- ));
7930		- } else {
7931		- // Mark for retry
7932		- $wpdb->update(
7933		- $table_name,
7934		- array(
7935		- 'status' => 'failed',
7936		- 'error_message' => $error_message
7937		- ),
7938		- array('id' => $item_id),
7939		- array('%s', '%s'),
7940		- array('%d')
7941		- );
7942		-
7943		- wp_send_json_error(array(
7944		- 'message' => 'Item processing failed, will retry: ' . $error_message,
7945		- 'can_retry' => true,
7946		- 'item_id' => $item_id,
7947		- 'attempts' => $item ? $item->attempts : 0
7948		- ));
7949		- }
7950		- }
7951		-}
7952		-
7953		-/**
7954		- * Process a URL from the queue
7955		- */
7956		-private function mxchat_process_queue_url($item_data, $bot_id = 'default', $queue_id = '') {
7957		- $url = isset($item_data['url']) ? $item_data['url'] : '';
7958		-
7959		- if (empty($url)) {
7960		- return new WP_Error('invalid_url', 'URL is empty');
7961		- }
7962		-
7963		- // Get bot-specific API key early (needed for both paths)
7964		- $bot_options = $this->get_bot_options($bot_id);
7965		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
7966		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
7967		-
7968		- if (strpos($selected_model, 'voyage') === 0) {
7969		- $api_key = $options['voyage_api_key'] ?? '';
7970		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
7971		- $api_key = $options['gemini_api_key'] ?? '';
7972		- } else {
7973		- $api_key = $options['api_key'] ?? '';
7974		- }
7975		-
7976		- if (empty($api_key)) {
7977		- return new WP_Error('no_api_key', 'API key not configured for bot: ' . $bot_id);
7978		- }
7979		-
7980		- // Check if this is a WooCommerce product URL and WooCommerce is active
7981		- $is_product_url = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
7982		- $content_type = $is_product_url ? 'product' : 'url';
7983		-
7984		- // Try to get WooCommerce product data if it's a product URL
7985		- if ($is_product_url && class_exists('WooCommerce')) {
7986		- $product_content = $this->mxchat_extract_woocommerce_product_content($url);
7987		-
7988		- if (!empty($product_content)) {
7989		- // Successfully extracted WooCommerce product data with pricing
7990		- $result = MxChat_Utils::submit_content_to_db(
7991		- $product_content,
7992		- $url,
7993		- $api_key,
7994		- null,
7995		- $bot_id,
7996		- 'product'
7997		- );
7998		- return $result;
7999		- }
8000		- // If WooCommerce extraction failed, fall through to HTML extraction
8001		- }
8002		-
8003		- // Fetch URL content (fallback for non-products or when WooCommerce extraction fails)
8004		- $is_likely_pdf = (strtolower(pathinfo(parse_url($url, PHP_URL_PATH) ?: '', PATHINFO_EXTENSION)) === 'pdf');
8005		- $response = wp_remote_get($url, array(
8006		- 'timeout' => $is_likely_pdf ? 120 : 30,
8007		- 'redirection' => 5,
8008		- 'user-agent' => mxchat_ingest_user_agent(),
8009		- ));
8010		-
8011		- if (is_wp_error($response)) {
8012		- return $response;
8013		- }
8014		-
8015		- $response_code = wp_remote_retrieve_response_code($response);
8016		- if ($response_code !== 200) {
8017		- return new WP_Error('http_error', 'HTTP ' . $response_code . ' error for: ' . $url);
8018		- }
8019		-
8020		- // Check if URL is a PDF — expand into per-page queue items using the standard PDF pipeline
8021		- if ($this->mxchat_is_pdf_url($url, $response)) {
8022		- return $this->mxchat_expand_pdf_to_queue($url, $response, $bot_id, $queue_id);
8023		- }
8024		-
8025		- $html = wp_remote_retrieve_body($response);
8026		-
8027		- if (empty($html)) {
8028		- return new WP_Error('empty_response', 'Empty response body');
8029		- }
8030		-
8031		- // Extract and sanitize content
8032		- $content = $this->mxchat_extract_main_content($html);
8033		- $sanitized = $this->mxchat_sanitize_content_for_api($content);
8034		-
8035		- if (empty($sanitized)) {
8036		- // Not an error - just no content found (maybe a redirect or empty page)
8037		- return false;
8038		- }
8039		-
8040		- // Submit to database with content_type
8041		- $result = MxChat_Utils::submit_content_to_db(
8042		- $sanitized,
8043		- $url,
8044		- $api_key,
8045		- null,
8046		- $bot_id,
8047		- $content_type
8048		- );
8049		-
8050		- return $result;
8051		-}
8052		-
8053		-/**
8054		- * Expand a PDF URL into per-page queue items using the standard PDF pipeline.
8055		- * Called when a sitemap URL turns out to be a PDF — downloads, parses page count,
8056		- * and adds pdf_page items to the same queue so they process with full progress tracking.
8057		- */
8058		-private function mxchat_expand_pdf_to_queue($pdf_url, $response, $bot_id = 'default', $queue_id = '') {
8059		- set_time_limit(120); // PDFs need extra time for download + parsing
8060		-
8061		- $upload_dir = wp_upload_dir();
8062		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
8063		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
8064		-
8065		- $response_body = wp_remote_retrieve_body($response);
8066		- if (empty($response_body)) {
8067		- return new WP_Error('empty_pdf', 'Empty PDF response for: ' . $pdf_url);
8068		- }
8069		-
8070		- if (!wp_mkdir_p(dirname($pdf_path))) {
8071		- return new WP_Error('dir_error', 'Failed to create upload directory');
8072		- }
8073		-
8074		- file_put_contents($pdf_path, $response_body);
8075		-
8076		- if (!file_exists($pdf_path)) {
8077		- return new WP_Error('save_error', 'Failed to save PDF file');
8078		- }
8079		-
8080		- try {
8081		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
8082		-
8083		- if ($total_pages === false || $total_pages < 1) {
8084		- wp_delete_file($pdf_path);
8085		- return new WP_Error('no_pages', 'PDF has no pages: ' . $pdf_url);
8086		- }
8087		-
8088		- // Build per-page items identical to mxchat_handle_pdf_for_knowledge_base
8089		- $pages = array();
8090		- for ($i = 1; $i <= $total_pages; $i++) {
8091		- $pages[] = array(
8092		- 'pdf_path' => $pdf_path,
8093		- 'pdf_url' => $pdf_url,
8094		- 'page_number' => $i,
8095		- 'total_pages' => $total_pages
8096		- );
8097		- }
8098		-
8099		- // Add pdf_page items to the SAME queue so the JS picks them up automatically
8100		- if (!empty($queue_id)) {
8101		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
8102		- } else {
8103		- // Fallback: create a new PDF queue (shouldn't happen in sitemap flow)
8104		- $new_queue_id = 'pdf_' . md5($pdf_url . time());
8105		- $queued_count = $this->mxchat_add_to_queue($new_queue_id, 'pdf_page', $pages, $bot_id);
8106		- $this->mxchat_set_queue_meta($new_queue_id, 'source_url', $pdf_url);
8107		- $this->mxchat_set_queue_meta($new_queue_id, 'queue_type', 'pdf');
8108		- $this->mxchat_set_queue_meta($new_queue_id, 'total_items', $total_pages);
8109		- $this->mxchat_set_queue_meta($new_queue_id, 'bot_id', $bot_id);
8110		- $this->mxchat_set_queue_meta($new_queue_id, 'pdf_path', $pdf_path);
8111		- $this->mxchat_set_queue_meta($new_queue_id, 'created_at', current_time('mysql'));
8112		- }
8113		-
8114		- if ($queued_count === 0) {
8115		- wp_delete_file($pdf_path);
8116		- return new WP_Error('queue_error', 'Failed to add PDF pages to queue');
8117		- }
8118		-
8119		- // Return true so the original URL item is marked complete
8120		- // The new pdf_page items will be processed in subsequent batches
8121		- return true;
8122		-
8123		- } catch (Exception $e) {
8124		- if (file_exists($pdf_path)) {
8125		- wp_delete_file($pdf_path);
8126		- }
8127		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
8128		- }
8129		-}
8130		-
8131		-/**
8132		- * Legacy: Process a PDF URL inline during sitemap queue processing.
8133		- * @deprecated Use mxchat_expand_pdf_to_queue instead — kept for reference only.
8134		- */
8135		-private function mxchat_process_pdf_url_inline($pdf_url, $response, $api_key, $bot_id = 'default') {
8136		- set_time_limit(120); // PDFs need more time — downloading + parsing all pages
8137		-
8138		- $upload_dir = wp_upload_dir();
8139		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
8140		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
8141		-
8142		- $response_body = wp_remote_retrieve_body($response);
8143		- if (empty($response_body)) {
8144		- return new WP_Error('empty_pdf', 'Empty PDF response');
8145		- }
8146		-
8147		- if (!wp_mkdir_p(dirname($pdf_path))) {
8148		- return new WP_Error('dir_error', 'Failed to create upload directory');
8149		- }
8150		-
8151		- file_put_contents($pdf_path, $response_body);
8152		-
8153		- if (!file_exists($pdf_path)) {
8154		- return new WP_Error('save_error', 'Failed to save PDF file');
8155		- }
8156		-
8157		- try {
8158		- mxchat_load_pdf_parser();
8159		- $parser = new \Smalot\PdfParser\Parser();
8160		- $pdf = $parser->parseFile($pdf_path);
8161		- $pages = $pdf->getPages();
8162		- $total_pages = count($pages);
8163		-
8164		- if ($total_pages < 1) {
8165		- wp_delete_file($pdf_path);
8166		- return new WP_Error('no_pages', 'PDF has no pages');
8167		- }
8168		-
8169		- $processed = 0;
8170		- $skipped_pages = array();
8171		-
8172		- for ($i = 0; $i < $total_pages; $i++) {
8173		- $page_num = $i + 1;
8174		- $text = $pages[$i]->getText();
8175		- if (empty($text)) {
8176		- $skipped_pages[] = 'Page ' . $page_num . ': No text could be extracted — page may contain only images, links, or non-standard encoding';
8177		- continue;
8178		- }
8179		-
8180		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
8181		- if (empty($sanitized)) {
8182		- $skipped_pages[] = 'Page ' . $page_num . ': Text was extracted but contained only special characters, control codes, or unsupported content';
8183		- continue;
8184		- }
8185		-
8186		- $metadata = array(
8187		- 'document_type' => 'pdf',
8188		- 'total_pages' => $total_pages,
8189		- 'current_page' => $page_num,
8190		- 'source_url' => $pdf_url,
8191		- );
8192		-
8193		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
8194		- $page_url = esc_url($pdf_url . '#page=' . $page_num);
8195		-
8196		- MxChat_Utils::submit_content_to_db(
8197		- $content_with_metadata,
8198		- $page_url,
8199		- $api_key,
8200		- null,
8201		- $bot_id,
8202		- 'pdf'
8203		- );
8204		-
8205		- $processed++;
8206		- }
8207		-
8208		- // Clean up the temp PDF file
8209		- wp_delete_file($pdf_path);
8210		-
8211		- if (!empty($skipped_pages)) {
8212		- error_log('MxChat PDF: Skipped ' . count($skipped_pages) . ' of ' . $total_pages . ' pages: ' . implode('; ', $skipped_pages));
8213		- }
8214		-
8215		- return $processed > 0 ? true : false;
8216		-
8217		- } catch (Exception $e) {
8218		- if (file_exists($pdf_path)) {
8219		- wp_delete_file($pdf_path);
8220		- }
8221		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
8222		- }
8223		-}
8224		-
8225		-/**
8226		- * Extract WooCommerce product content including pricing
8227		- *
8228		- * @param string $url The product URL
8229		- * @return string|false Product content with pricing, or false if not found
8230		- */
8231		-private function mxchat_extract_woocommerce_product_content($url) {
8232		- // Try to get product ID from URL
8233		- $product_id = url_to_postid($url);
8234		-
8235		- // If url_to_postid fails, try to extract from URL pattern
8236		- if (!$product_id) {
8237		- $product_slug = '';
8238		-
8239		- // Handle pretty permalinks: /product/product-name/
8240		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
8241		- $product_slug = $matches[1];
8242		- }
8243		-
8244		- if (!empty($product_slug)) {
8245		- $product_post = get_page_by_path($product_slug, OBJECT, 'product');
8246		- if ($product_post) {
8247		- $product_id = $product_post->ID;
8248		- }
8249		- }
8250		- }
8251		-
8252		- if (!$product_id) {
8253		- return false;
8254		- }
8255		-
8256		- // Get WooCommerce product object
8257		- $product = wc_get_product($product_id);
8258		-
8259		- if (!$product) {
8260		- return false;
8261		- }
8262		-
8263		- // Build product content with pricing (similar to mxchat_store_product_embedding)
8264		- $title = $product->get_name();
8265		- $description = $product->get_description();
8266		- $short_description = $product->get_short_description();
8267		- $sku = $product->get_sku();
8268		-
8269		- // Get pricing information
8270		- $regular_price = $product->get_regular_price();
8271		- $sale_price = $product->get_sale_price();
8272		- $price = $product->get_price(); // Current active price
8273		-
8274		- // Get currency symbol
8275		- $currency_symbol = get_woocommerce_currency_symbol();
8276		-
8277		- // Format content
8278		- $content = $title . "\n\n";
8279		-
8280		- if (!empty($short_description)) {
8281		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
8282		- }
8283		-
8284		- if (!empty($description)) {
8285		- $content .= wp_strip_all_tags($description) . "\n\n";
8286		- }
8287		-
8288		- // Add pricing information
8289		- if (!empty($regular_price)) {
8290		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
8291		- } elseif (!empty($price)) {
8292		- $content .= "Price: " . $currency_symbol . $price . "\n";
8293		- }
8294		-
8295		- if (!empty($sale_price) && $sale_price !== $regular_price) {
8296		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
8297		- }
8298		-
8299		- // Handle variable products - show price range
8300		- if ($product->is_type('variable')) {
8301		- $min_price = $product->get_variation_price('min');
8302		- $max_price = $product->get_variation_price('max');
8303		- if ($min_price !== $max_price) {
8304		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
8305		- }
8306		- }
8307		-
8308		- if (!empty($sku)) {
8309		- $content .= "SKU: " . $sku . "\n";
8310		- }
8311		-
8312		- // Get product categories
8313		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
8314		- if (!empty($categories) && !is_wp_error($categories)) {
8315		- $content .= "Categories: " . implode(', ', $categories) . "\n";
8316		- }
8317		-
8318		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
8319		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
8320		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
8321		- foreach ($custom_tabs as $tab) {
8322		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
8323		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
8324		-
8325		- if (!empty($tab_title) && !empty($tab_content)) {
8326		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
8327		- }
8328		- }
8329		- }
8330		-
8331		- // Also check for reusable/saved tabs applied to this product
8332		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
8333		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
8334		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
8335		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
8336		- foreach ($applied_saved_tabs as $saved_tab_id) {
8337		- if (isset($saved_tabs[$saved_tab_id])) {
8338		- $tab = $saved_tabs[$saved_tab_id];
8339		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
8340		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
8341		-
8342		- if (!empty($tab_title) && !empty($tab_content)) {
8343		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
8344		- }
8345		- }
8346		- }
8347		- }
8348		- }
8349		-
8350		- return $this->mxchat_sanitize_content_for_api($content);
8351		-}
8352		-
8353		-/**
8354		- * Process a PDF page from the queue
8355		- */
8356		-private function mxchat_process_queue_pdf_page($item_data, $bot_id = 'default') {
8357		- $pdf_path = isset($item_data['pdf_path']) ? $item_data['pdf_path'] : '';
8358		- $pdf_url = isset($item_data['pdf_url']) ? $item_data['pdf_url'] : '';
8359		- $page_number = isset($item_data['page_number']) ? absint($item_data['page_number']) : 0;
8360		- $total_pages = isset($item_data['total_pages']) ? absint($item_data['total_pages']) : 0;
8361		-
8362		- if (empty($pdf_path) || !file_exists($pdf_path)) {
8363		- return new WP_Error('pdf_not_found', 'PDF file not found: ' . $pdf_path);
8364		- }
8365		-
8366		- if ($page_number < 1) {
8367		- return new WP_Error('invalid_page', 'Invalid page number');
8368		- }
8369		-
8370		- try {
8371		- mxchat_load_pdf_parser();
8372		- $parser = new \Smalot\PdfParser\Parser();
8373		- $pdf = $parser->parseFile($pdf_path);
8374		- $pages = $pdf->getPages();
8375		-
8376		- if (!isset($pages[$page_number - 1])) {
8377		- return new WP_Error('page_not_found', 'Page ' . $page_number . ' not found in PDF');
8378		- }
8379		-
8380		- $text = $pages[$page_number - 1]->getText();
8381		-
8382		- if (empty($text)) {
8383		- return new WP_Error('empty_page', 'Page ' . $page_number . ': No text could be extracted — page may contain only images, links, or non-standard encoding');
8384		- }
8385		-
8386		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
8387		-
8388		- if (empty($sanitized)) {
8389		- return new WP_Error('empty_after_sanitization', 'Page ' . $page_number . ': Text was extracted but contained only special characters, control codes, or unsupported content that was removed during cleanup');
8390		- }
8391		-
8392		- // Create metadata
8393		- $metadata = array(
8394		- 'document_type' => 'pdf',
8395		- 'total_pages' => $total_pages,
8396		- 'current_page' => $page_number,
8397		- 'source_url' => $pdf_url
8398		- );
8399		-
8400		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
8401		- $page_url = esc_url($pdf_url . "#page=" . $page_number);
8402		-
8403		- // Get bot-specific API key
8404		- $bot_options = $this->get_bot_options($bot_id);
8405		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
8406		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
8407		-
8408		- if (strpos($selected_model, 'voyage') === 0) {
8409		- $api_key = $options['voyage_api_key'] ?? '';
8410		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
8411		- $api_key = $options['gemini_api_key'] ?? '';
8412		- } else {
8413		- $api_key = $options['api_key'] ?? '';
8414		- }
8415		-
8416		- if (empty($api_key)) {
8417		- return new WP_Error('no_api_key', 'API key not configured for bot: ' . $bot_id);
8418		- }
8419		-
8420		- // Submit to database - UPDATED 2.5.6: Added content_type 'pdf'
8421		- $result = MxChat_Utils::submit_content_to_db(
8422		- $content_with_metadata,
8423		- $page_url,
8424		- $api_key,
8425		- null,
8426		- $bot_id,
8427		- 'pdf'
8428		- );
8429		-
8430		- return $result;
8431		-
8432		- } catch (Exception $e) {
8433		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
8434		- }
8435		-}
8436		-
8437		-/**
8438		- * AJAX: Get queue processing status
8439		- */
8440		-public function ajax_mxchat_get_queue_status() {
8441		- // Verify nonce and permissions
8442		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8443		-
8444		- if (!current_user_can('manage_options')) {
8445		- wp_send_json_error('Unauthorized access');
8446		- }
8447		-
8448		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8449		-
8450		- if (empty($queue_id)) {
8451		- wp_send_json_error('Missing queue ID');
8452		- }
8453		-
8454		- global $wpdb;
8455		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8456		-
8457		- // Get counts by status
8458		- $counts = $wpdb->get_results($wpdb->prepare(
8459		- "SELECT status, COUNT(*) as count
8460		- FROM $table_name
8461		- WHERE queue_id = %s
8462		- GROUP BY status",
8463		- $queue_id
8464		- ), OBJECT_K);
8465		-
8466		- $total = 0;
8467		- $completed = 0;
8468		- $failed = 0;
8469		- $processing = 0;
8470		- $pending = 0;
8471		-
8472		- foreach ($counts as $status => $data) {
8473		- $count = absint($data->count);
8474		- $total += $count;
8475		-
8476		- switch ($status) {
8477		- case 'completed':
8478		- $completed = $count;
8479		- break;
8480		- case 'failed':
8481		- $failed = $count;
8482		- break;
8483		- case 'processing':
8484		- $processing = $count;
8485		- break;
8486		- case 'pending':
8487		- $pending = $count;
8488		- break;
8489		- }
8490		- }
8491		-
8492		- // Calculate percentage
8493		- $percentage = $total > 0 ? round((($completed + $failed) / $total) * 100) : 0;
8494		-
8495		- // Get failed items details (include all failed items, not just those that exhausted retries)
8496		- $failed_items = array();
8497		- if ($failed > 0) {
8498		- $failed_items = $wpdb->get_results($wpdb->prepare(
8499		- "SELECT item_type, item_data, error_message, attempts
8500		- FROM $table_name
8501		- WHERE queue_id = %s
8502		- AND status = 'failed'
8503		- ORDER BY id DESC
8504		- LIMIT 50",
8505		- $queue_id
8506		- ));
8507		- }
8508		-
8509		- // Get queue metadata
8510		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
8511		- $queue_type = $this->mxchat_get_queue_meta($queue_id, 'queue_type');
8512		-
8513		- // Determine if queue is complete
8514		- $is_complete = ($pending === 0 && $processing === 0);
8515		-
8516		- wp_send_json_success(array(
8517		- 'queue_id' => $queue_id,
8518		- 'queue_type' => $queue_type,
8519		- 'source_url' => $source_url,
8520		- 'total' => $total,
8521		- 'completed' => $completed,
8522		- 'failed' => $failed,
8523		- 'processing' => $processing,
8524		- 'pending' => $pending,
8525		- 'percentage' => $percentage,
8526		- 'is_complete' => $is_complete,
8527		- 'failed_items' => $failed_items,
8528		- 'status' => $is_complete ? 'complete' : 'processing'
8529		- ));
8530		-}
8531		-
8532		-/**
8533		- * AJAX: Clear completed queue
8534		- */
8535		-public function ajax_mxchat_clear_queue() {
8536		- // Verify nonce and permissions
8537		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8538		-
8539		- if (!current_user_can('manage_options')) {
8540		- wp_send_json_error('Unauthorized access');
8541		- }
8542		-
8543		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8544		-
8545		- if (empty($queue_id)) {
8546		- wp_send_json_error('Missing queue ID');
8547		- }
8548		-
8549		- global $wpdb;
8550		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8551		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
8552		-
8553		- // Delete queue items
8554		- $wpdb->delete(
8555		- $table_name,
8556		- array('queue_id' => $queue_id),
8557		- array('%s')
8558		- );
8559		-
8560		- // Delete queue metadata
8561		- $wpdb->delete(
8562		- $meta_table,
8563		- array('queue_id' => $queue_id),
8564		- array('%s')
8565		- );
8566		-
8567		- wp_send_json_success(array(
8568		- 'message' => 'Queue cleared successfully'
8569		- ));
8570		-}
8571		-
8572		-/**
8573		- * AJAX: Retry failed items in queue
8574		- */
8575		-public function ajax_mxchat_retry_failed() {
8576		- // Verify nonce and permissions
8577		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8578		-
8579		- if (!current_user_can('manage_options')) {
8580		- wp_send_json_error('Unauthorized access');
8581		- }
8582		-
8583		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8584		-
8585		- if (empty($queue_id)) {
8586		- wp_send_json_error('Missing queue ID');
8587		- }
8588		-
8589		- global $wpdb;
8590		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8591		-
8592		- // Reset failed items to pending and reset attempt count
8593		- $updated = $wpdb->update(
8594		- $table_name,
8595		- array(
8596		- 'status' => 'pending',
8597		- 'attempts' => 0,
8598		- 'error_message' => null
8599		- ),
8600		- array(
8601		- 'queue_id' => $queue_id,
8602		- 'status' => 'failed'
8603		- ),
8604		- array('%s', '%d', '%s'),
8605		- array('%s', '%s')
8606		- );
8607		-
8608		- wp_send_json_success(array(
8609		- 'message' => 'Reset ' . $updated . ' failed items for retry',
8610		- 'reset_count' => $updated
8611		- ));
8612		-}
8613		-
8614		-
8615		-public function ajax_mxchat_mark_queue_complete() {
8616		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8617		-
8618		- if (!current_user_can('manage_options')) {
8619		- wp_send_json_error('Unauthorized access');
8620		- }
8621		-
8622		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8623		-
8624		- if (empty($queue_id)) {
8625		- wp_send_json_error('Missing queue ID');
8626		- }
8627		-
8628		- // Clear active queue transients
8629		- if (strpos($queue_id, 'sitemap_') === 0) {
8630		- delete_transient('mxchat_active_queue_sitemap');
8631		- } else if (strpos($queue_id, 'pdf_') === 0) {
8632		- delete_transient('mxchat_active_queue_pdf');
8633		- }
8634		-
8635		- wp_send_json_success(array('message' => 'Queue marked as complete'));
8636		-}
8637		-
8638	3553
8639	3554	// ========================================
8640	3555	// STATIC ACCESS METHODS
8641	3556	// ========================================