← All changes | admin/class-knowledge-manager.php +1739 -7663 3.2.21 → 2.2.7 View file →

		@@ -9,111 +9,57 @@
9	9	exit; // Exit if accessed directly
10	10	}
11	11
12	12	class MxChat_Knowledge_Manager {
13		-
	13	+
14	14	private $options;
15		-
16		- // Post IDs whose vectors were already deleted by mxchat_handle_status_transition this
17		- // request, so the transient-based branch in mxchat_handle_post_update can skip the
18		- // redundant (idempotent but network-visible) second deletion.
19		- private $transition_deleted_posts = array();
20		-
21		- // Post IDs already INDEXED by mxchat_handle_status_transition's arrival edge this
22		- // request. Normal editor publishes fire transition_post_status first, then
23		- // post_updated — without this guard every editor publish would embed twice.
24		- private $transition_indexed_posts = array();
25		-
26		- // Post IDs core has announced an in-flight UPDATE for. pre_post_update fires only
27		- // inside wp_insert_post's update branch and always before wp_transition_post_status,
28		- // so this is an exact "a post_updated is coming later this request" signal — which is
29		- // what makes it safe to arm transition_indexed_posts (plan a664f3).
30		- private $pending_post_update = array();
31		-
	15	+
32	16	/**
33	17	* Constructor - Register hooks for content processing
34	18	*/
35		-public function __construct() {
36		- $this->options = get_option('mxchat_options', array());
37		- $this->mxchat_init_hooks();
	19	+ public function __construct() {
	20	+ $this->options = get_option('mxchat_options', array());
	21	+ $this->mxchat_init_hooks();
	22	+ }
38	23
39		- $this->mxchat_init_role_hooks();
40		-}
41		-
42		-/**
43		- * Initialize WordPress hooks for content processing
44		- *
45		- */
46		-private function mxchat_init_hooks() {
47		- // Admin post handlers for form submissions
48		- add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
49		- add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
50		- add_action('admin_post_mxchat_submit_pdf_file', array($this, 'mxchat_handle_pdf_file_submission'));
51		- add_action('admin_post_mxchat_submit_document_file', array($this, 'mxchat_handle_document_file_submission'));
52		- add_action('admin_post_mxchat_submit_youtube', array($this, 'mxchat_handle_youtube_submission'));
53		- add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
54		-
55		- // AJAX handlers for real-time processing and status updates
56		- add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
57		- add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status'));
58		- add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
59		- add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
60		- add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
61		- add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
62		- add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
63		- add_action('wp_ajax_mxchat_delete_chunks_by_url', array($this, 'ajax_mxchat_delete_chunks_by_url'));
64		- add_action('wp_ajax_mxchat_delete_wordpress_prompt', array($this, 'ajax_mxchat_delete_wordpress_prompt'));
65		- add_action('wp_ajax_mxchat_bulk_delete_knowledge', array($this, 'ajax_mxchat_bulk_delete_knowledge'));
66		- add_action('wp_ajax_mxchat_update_role_restriction', array($this, 'ajax_mxchat_update_role_restriction'));
67		-
68		- // Queue-based processing AJAX handlers
69		- add_action('wp_ajax_mxchat_get_next_queue_item', array($this, 'ajax_mxchat_get_next_queue_item'));
70		- add_action('wp_ajax_mxchat_process_queue_item', array($this, 'ajax_mxchat_process_queue_item'));
71		- add_action('wp_ajax_mxchat_get_queue_status', array($this, 'ajax_mxchat_get_queue_status'));
72		- add_action('wp_ajax_mxchat_clear_queue', array($this, 'ajax_mxchat_clear_queue'));
73		- add_action('wp_ajax_mxchat_retry_failed', array($this, 'ajax_mxchat_retry_failed'));
74		- add_action('wp_ajax_mxchat_get_recent_entries', array($this, 'ajax_mxchat_get_recent_entries'));
75		- add_action('wp_ajax_mxchat_detect_sitemaps', array($this, 'ajax_mxchat_detect_sitemaps'));
76		- add_action('wp_ajax_mxchat_refresh_pinecone_entries', array($this, 'ajax_mxchat_refresh_pinecone_entries'));
77		- add_action('wp_ajax_mxchat_paginate_entries', array($this, 'ajax_mxchat_paginate_entries'));
78		- add_action('wp_ajax_mxchat_get_entry_content', array($this, 'ajax_mxchat_get_entry_content'));
79		- add_action('wp_ajax_mxchat_save_entry_content', array($this, 'ajax_mxchat_save_entry_content'));
80		- add_action('wp_ajax_mxchat_inspect_entry', array($this, 'ajax_mxchat_inspect_entry'));
	24	+ /**
	25	+ * Initialize WordPress hooks for content processing
	26	+ */
	27	+ private function mxchat_init_hooks() {
	28	+ // Admin post handlers for form submissions
	29	+ add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
	30	+ add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
	31	+ add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
	32	+
	33	+ // AJAX handlers for real-time processing and status updates
	34	+ add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
	35	+ add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status')); // NEW
	36	+ add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
	37	+ add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
	38	+ add_action('wp_ajax_mxchat_manual_batch_process', array($this, 'ajax_manual_batch_process'));
	39	+ add_action('mxchat_delete_content', array($this, 'mxchat_delete_from_pinecone_by_url'), 10, 1);
81	40
82		- // WordPress post management hooks
83		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
84		- add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
85		- add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
86		- add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
87		- // Authoritative unpublish detection: core hands this hook the REAL previous status, so
88		- // removal no longer depends on the mxchat_prev_status_* transients (evictable by persistent
89		- // object caches, never written by paths that bypass wp_update_post, e.g. plugins flipping
90		- // post_status directly and calling wp_transition_post_status themselves).
91		- add_action('transition_post_status', array($this, 'mxchat_handle_status_transition'), 10, 3);
92		-
93		- // ACF hook - fires AFTER ACF fields are saved, ensuring ACF data is available
94		- // Priority 20 to run after ACF's own save (which runs at priority 10)
95		- add_action('acf/save_post', array($this, 'mxchat_handle_acf_save'), 20);
96		-
97		- // One-time cleanup for vectors orphaned by unpublishes that predate the
98		- // transition_post_status handler (plan 816fb1): wp mxchat prune-unpublished
99		- if (defined('WP_CLI') && WP_CLI) {
100		- WP_CLI::add_command('mxchat prune-unpublished', array($this, 'cli_prune_unpublished'));
101		- // In-place repair for RTL KB rows imported in visual order before the
102		- // 32bf9e normalizer existed: wp mxchat rtl-repair (plan d1e6f7)
103		- WP_CLI::add_command('mxchat rtl-repair', array($this, 'cli_rtl_repair'));
	41	+
	42	+ // Cron handlers for background processing
	43	+ add_action('mxchat_process_sitemap_urls', array($this, 'mxchat_process_sitemap_urls_cron'), 10, 5);
	44	+ add_action('mxchat_process_pdf_pages', array($this, 'mxchat_process_pdf_pages_cron'), 10, 5);
	45	+
	46	+ // WordPress post management hooks
	47	+ add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
	48	+ add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
	49	+ add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
	50	+ add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
	51	+ add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
	52	+ add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
	53	+
	54	+ // WooCommerce product hooks (if WooCommerce is active)
	55	+ if (class_exists('WooCommerce')) {
	56	+ add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
	57	+ add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
	58	+ add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
	59	+ }
	60	+
104	61	}
105		-
106		- add_action('wp_ajax_mxchat_mark_queue_complete', array($this, 'ajax_mxchat_mark_queue_complete'));
107		-
108		- // WooCommerce product hooks (if WooCommerce is active)
109		- if (class_exists('WooCommerce')) {
110		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
111		- add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
112		- add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
113		- add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
114		- }
115		-}
116	62
117	63	/**
118	64	* Get current options (refreshed)
119	65	*/
		@@ -123,9 +69,315 @@
123	69	}
124	70	return $this->options;
125	71	}
126	72
	73	+
	74	+ /**
	75	+ * Handle manual batch processing via AJAX
	76	+ */
	77	+public function ajax_manual_batch_process() {
	78	+ try {
	79	+ // Verify nonce and permissions
	80	+ check_ajax_referer('mxchat_status_nonce', 'nonce');
	81	+
	82	+ if (!current_user_can('manage_options')) {
	83	+ wp_send_json_error('Unauthorized access');
	84	+ }
	85	+
	86	+ $process_type = sanitize_text_field($_POST['process_type'] ?? '');
	87	+ $url = sanitize_text_field($_POST['url'] ?? '');
	88	+
	89	+ if (empty($process_type) || empty($url)) {
	90	+ wp_send_json_error('Missing required parameters');
	91	+ }
	92	+
	93	+ $processed = 0;
	94	+
	95	+ if ($process_type === 'pdf') {
	96	+ $processed = $this->mxchat_manual_process_pdf_batch($url);
	97	+ } elseif ($process_type === 'sitemap') {
	98	+ $processed = $this->mxchat_manual_process_sitemap_batch($url);
	99	+ }
	100	+
	101	+ if ($processed > 0) {
	102	+ wp_send_json_success(array(
	103	+ 'message' => "Processed {$processed} items successfully",
	104	+ 'processed' => $processed
	105	+ ));
	106	+ } else {
	107	+ wp_send_json_error('No items were processed');
	108	+ }
	109	+
	110	+ } catch (Exception $e) {
	111	+ //error_log('Manual batch process error: ' . $e->getMessage());
	112	+ wp_send_json_error('Processing failed: ' . $e->getMessage());
	113	+ }
	114	+}
127	115
	116	+/**
	117	+ * Process a small PDF batch manually - DIRECT PROCESSING
	118	+ */
	119	+private function mxchat_manual_process_pdf_batch($pdf_url) {
	120	+ try {
	121	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	122	+ $status = get_transient($status_key);
	123	+
	124	+ if (!$status || $status['status'] !== 'processing') {
	125	+ //error_log('Manual PDF: No processing status found');
	126	+ return 0;
	127	+ }
	128	+
	129	+ //error_log('Manual PDF: Starting direct processing for ' . $pdf_url);
	130	+
	131	+ // Get current progress
	132	+ $current_page = $status['processed_pages'] ?? 0;
	133	+ $total_pages = $status['total_pages'] ?? 0;
	134	+
	135	+ if ($current_page >= $total_pages) {
	136	+ //error_log('Manual PDF: Already completed');
	137	+ return 0;
	138	+ }
	139	+
	140	+ // Try to download the PDF again for processing
	141	+ $response = wp_remote_get($pdf_url, array('timeout' => 30));
	142	+
	143	+ if (is_wp_error($response)) {
	144	+ //error_log('Manual PDF: Failed to download PDF: ' . $response->get_error_message());
	145	+ return 0;
	146	+ }
	147	+
	148	+ $pdf_content = wp_remote_retrieve_body($response);
	149	+ if (empty($pdf_content)) {
	150	+ //error_log('Manual PDF: Empty PDF content');
	151	+ return 0;
	152	+ }
	153	+
	154	+ // Save PDF temporarily
	155	+ $upload_dir = wp_upload_dir();
	156	+ $temp_pdf_path = trailingslashit($upload_dir['path']) . 'temp_manual_' . time() . '.pdf';
	157	+ file_put_contents($temp_pdf_path, $pdf_content);
	158	+
	159	+ // Process 2 pages directly
	160	+ $processed = $this->mxchat_process_pdf_pages_direct($temp_pdf_path, $pdf_url, $current_page, 5);
	161	+
	162	+ // Clean up temp file
	163	+ if (file_exists($temp_pdf_path)) {
	164	+ wp_delete_file($temp_pdf_path);
	165	+ }
	166	+
	167	+ //error_log('Manual PDF: Processed ' . $processed . ' pages');
	168	+ return $processed;
	169	+
	170	+ } catch (Exception $e) {
	171	+ //error_log('Manual PDF batch error: ' . $e->getMessage());
	172	+ return 0;
	173	+ }
	174	+}
	175	+
	176	+/**
	177	+ * Process PDF pages directly without cron
	178	+ */
	179	+private function mxchat_process_pdf_pages_direct($pdf_path, $pdf_url, $start_page, $batch_size) {
	180	+ try {
	181	+ if (!file_exists($pdf_path)) {
	182	+ //error_log('Direct PDF: File not found at ' . $pdf_path);
	183	+ return 0;
	184	+ }
	185	+
	186	+ $parser = new \Smalot\PdfParser\Parser();
	187	+ $pdf = $parser->parseFile($pdf_path);
	188	+ $pages = $pdf->getPages();
	189	+
	190	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	191	+ $status = get_transient($status_key);
	192	+
	193	+ if (!$status) {
	194	+ return 0;
	195	+ }
	196	+
	197	+ $options = get_option('mxchat_options');
	198	+ $api_key = $options['api_key'] ?? '';
	199	+
	200	+ if (empty($api_key)) {
	201	+ //error_log('Direct PDF: No API key');
	202	+ return 0;
	203	+ }
	204	+
	205	+ $processed = 0;
	206	+ $end_page = min($start_page + $batch_size, count($pages));
	207	+
	208	+ for ($i = $start_page; $i < $end_page; $i++) {
	209	+ try {
	210	+ $page_number = $i + 1;
	211	+ $text = $pages[$i]->getText();
	212	+
	213	+ if (empty($text)) {
	214	+ //error_log('Direct PDF: Empty text on page ' . $page_number);
	215	+ continue;
	216	+ }
	217	+
	218	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
	219	+ if (empty($sanitized_content)) {
	220	+ //error_log('Direct PDF: No content after sanitization on page ' . $page_number);
	221	+ continue;
	222	+ }
	223	+
	224	+ // Generate embedding
	225	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	226	+ if (is_string($embedding_vector)) {
	227	+ //error_log('Direct PDF: Embedding failed on page ' . $page_number . ': ' . $embedding_vector);
	228	+ continue;
	229	+ }
	230	+
	231	+ // Create metadata
	232	+ $metadata = array(
	233	+ 'document_type' => 'pdf',
	234	+ 'total_pages' => count($pages),
	235	+ 'current_page' => $page_number,
	236	+ 'source_url' => $pdf_url
	237	+ );
	238	+
	239	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	240	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
	241	+
	242	+ // Store in database
	243	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $api_key);
	244	+
	245	+ if (is_wp_error($db_result)) {
	246	+ //error_log('Direct PDF: DB error on page ' . $page_number . ': ' . $db_result->get_error_message());
	247	+ continue;
	248	+ }
	249	+
	250	+ $processed++;
	251	+ //error_log('Direct PDF: Successfully processed page ' . $page_number);
	252	+
	253	+ // Update status
	254	+ $status['processed_pages'] = $i + 1;
	255	+ $status['last_update'] = time();
	256	+ $status['percentage'] = round(($status['processed_pages'] / $status['total_pages']) * 100);
	257	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	258	+
	259	+ } catch (Exception $e) {
	260	+ //error_log('Direct PDF: Error processing page ' . ($i + 1) . ': ' . $e->getMessage());
	261	+ continue;
	262	+ }
	263	+ }
	264	+
	265	+ // Check if completed
	266	+ if ($status['processed_pages'] >= $status['total_pages']) {
	267	+ $status['status'] = 'complete';
	268	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	269	+ //error_log('Direct PDF: Processing completed');
	270	+ }
	271	+
	272	+ return $processed;
	273	+
	274	+ } catch (Exception $e) {
	275	+ //error_log('Direct PDF processing error: ' . $e->getMessage());
	276	+ return 0;
	277	+ }
	278	+}
	279	+
	280	+/**
	281	+ * Process a small sitemap batch manually - DIRECT PROCESSING
	282	+ */
	283	+private function mxchat_manual_process_sitemap_batch($sitemap_url) {
	284	+ try {
	285	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	286	+ $status = get_transient($status_key);
	287	+
	288	+ if (!$status || $status['status'] !== 'processing') {
	289	+ return 0;
	290	+ }
	291	+
	292	+ //error_log('Manual Sitemap: Starting direct processing for ' . $sitemap_url);
	293	+
	294	+ // Re-fetch the sitemap to get URLs
	295	+ $response = wp_remote_get($sitemap_url, array('timeout' => 30));
	296	+ if (is_wp_error($response)) {
	297	+ //error_log('Manual Sitemap: Failed to fetch sitemap');
	298	+ return 0;
	299	+ }
	300	+
	301	+ $sitemap_content = wp_remote_retrieve_body($response);
	302	+ $xml = simplexml_load_string($sitemap_content);
	303	+
	304	+ if (!$xml) {
	305	+ //error_log('Manual Sitemap: Invalid XML');
	306	+ return 0;
	307	+ }
	308	+
	309	+ $urls = array();
	310	+ foreach ($xml->url as $url_element) {
	311	+ $urls[] = (string)$url_element->loc;
	312	+ }
	313	+
	314	+ $current_processed = $status['processed_urls'] ?? 0;
	315	+ $batch_size = 5;
	316	+ $processed = 0;
	317	+
	318	+ // Process next 2 URLs
	319	+ for ($i = $current_processed; $i < min($current_processed + $batch_size, count($urls)); $i++) {
	320	+ $url = $urls[$i];
	321	+
	322	+ if ($this->mxchat_process_single_url_direct($url)) {
	323	+ $processed++;
	324	+ }
	325	+
	326	+ // Update status
	327	+ $status['processed_urls'] = $i + 1;
	328	+ $status['last_update'] = time();
	329	+ $status['percentage'] = round(($status['processed_urls'] / $status['total_urls']) * 100);
	330	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	331	+ }
	332	+
	333	+ // Check if completed
	334	+ if ($status['processed_urls'] >= $status['total_urls']) {
	335	+ $status['status'] = 'complete';
	336	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	337	+ }
	338	+
	339	+ //error_log('Manual Sitemap: Processed ' . $processed . ' URLs');
	340	+ return $processed;
	341	+
	342	+ } catch (Exception $e) {
	343	+ //error_log('Manual sitemap batch error: ' . $e->getMessage());
	344	+ return 0;
	345	+ }
	346	+}
	347	+
	348	+/**
	349	+ * Process a single URL directly
	350	+ */
	351	+private function mxchat_process_single_url_direct($url) {
	352	+ try {
	353	+ $response = wp_remote_get($url, array('timeout' => 30));
	354	+ if (is_wp_error($response)) {
	355	+ return false;
	356	+ }
	357	+
	358	+ $html = wp_remote_retrieve_body($response);
	359	+ $content = $this->mxchat_extract_main_content($html);
	360	+ $sanitized = $this->mxchat_sanitize_content_for_api($content);
	361	+
	362	+ if (empty($sanitized)) {
	363	+ return false;
	364	+ }
	365	+
	366	+ $options = get_option('mxchat_options');
	367	+ $api_key = $options['api_key'] ?? '';
	368	+
	369	+ $result = MxChat_Utils::submit_content_to_db($sanitized, $url, $api_key);
	370	+
	371	+ return !is_wp_error($result);
	372	+
	373	+ } catch (Exception $e) {
	374	+ //error_log('Single URL processing error: ' . $e->getMessage());
	375	+ return false;
	376	+ }
	377	+}
	378	+
	379	+
128	380	// ========================================
129	381	// MAIN CONTENT SUBMISSION HANDLERS
130	382	// ========================================
131	383
		@@ -141,30 +393,34 @@
141	393	wp_die(esc_html__('Nonce verification failed.', 'mxchat'));
142	394	}
143	395
144	396	// Sanitize the inputs.
145		- // Use wp_kses_post to allow safe HTML and wp_unslash to remove WordPress added slashes
146		- $article_content = wp_kses_post(wp_unslash($_POST['article_content']));
	397	+ $article_content = sanitize_textarea_field($_POST['article_content']);
147	398	$article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
148	399
149		- // Get bot_id from form submission
150		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
	400	+ // Get API key for submission
	401	+ $options = get_option('mxchat_options');
	402	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
151	403
152		- // Get bot-specific options and API key
153		- $bot_options = $this->get_bot_options($bot_id);
154		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
155		-
156		- // Custom-provider-aware decision; keyless custom sites must pass (plan cbd5fd).
157		- $preflight = MxChat_Utils::embedding_preflight($options);
158		- if (!$preflight['ok']) {
159		- set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
	404	+ if (strpos($selected_model, 'voyage') === 0) {
	405	+ $api_key = $options['voyage_api_key'] ?? '';
	406	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	407	+ $api_key = $options['gemini_api_key'] ?? '';
	408	+ } else {
	409	+ $api_key = $options['api_key'] ?? '';
	410	+ }
	411	+
	412	+ if (empty($api_key)) {
	413	+ set_transient('mxchat_admin_notice_error',
	414	+ esc_html__('API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
	415	+ 30
	416	+ );
160	417	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
161	418	exit;
162	419	}
163		- $api_key = $preflight['api_key'];
164	420
165		- // Use centralized utility function with bot_id
166		- $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key, null, $bot_id);
	421	+ // Use centralized utility function for storage
	422	+ $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key);
167	423
168	424	if (is_wp_error($result)) {
169	425	set_transient('mxchat_admin_notice_error',
170	426	esc_html__('Error storing content: ', 'mxchat') . $result->get_error_message(),
		@@ -179,253 +435,17 @@
179	435
180	436	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
181	437	exit;
182	438	}
183		-
184		-/**
185		- * Handle the "YouTube" KB import source (admin-post form submission).
186		- *
187		- * Per-video description mode:
188		- * - auto: fetch oEmbed metadata (reliable) + best-effort captions transcript.
189		- * If no usable transcript, index the metadata anyway, tell the admin,
190		- * and bounce back with the manual box pre-filled (never fail silently).
191		- * - manual: the admin's own description is what gets indexed; metadata rides along.
192		- *
193		- * The row is stored with content_type 'youtube' and source_url = the canonical
194		- * watch URL, so re-importing the same video UPDATES the entry (source_url
195		- * duplicate handling in MxChat_Utils::store_in_wordpress_db) — that is also the
196		- * "augment a metadata-only entry" path.
197		- */
198		-public function mxchat_handle_youtube_submission() {
199		- if (!isset($_POST['submit_youtube']) || !current_user_can('manage_options')) {
200		- wp_die(esc_html__('Unauthorized access', 'mxchat'));
201		- }
202		-
203		- check_admin_referer('mxchat_submit_youtube_action', 'mxchat_submit_youtube_nonce');
204		-
205		- $redirect_url = admin_url('admin.php?page=mxchat-prompts');
206		-
207		- $youtube_url = isset($_POST['youtube_url']) ? esc_url_raw(wp_unslash($_POST['youtube_url'])) : '';
208		- $video_id = MxChat_Utils::parse_youtube_id($youtube_url);
209		-
210		- if (empty($video_id)) {
211		- set_transient('mxchat_admin_notice_error',
212		- esc_html__('That does not look like a link to a single YouTube video. Please paste a watch, youtu.be, or Shorts URL.', 'mxchat'),
213		- 30
214		- );
215		- wp_safe_redirect(esc_url($redirect_url));
216		- exit;
217		- }
218		-
219		- $canonical_url = 'https://www.youtube.com/watch?v=' . $video_id;
220		-
221		- $description_mode = (isset($_POST['youtube_description_mode']) && $_POST['youtube_description_mode'] === 'manual') ? 'manual' : 'auto';
222		- $manual_description = isset($_POST['youtube_description']) ? trim(wp_kses_post(wp_unslash($_POST['youtube_description']))) : '';
223		-
224		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
225		-
226		- // Resolve the embedding decision exactly like the sibling handlers —
227		- // custom-provider-aware (plan cbd5fd).
228		- $bot_options = $this->get_bot_options($bot_id);
229		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
230		-
231		- $preflight = MxChat_Utils::embedding_preflight($options);
232		- if (!$preflight['ok']) {
233		- set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
234		- wp_safe_redirect(esc_url($redirect_url));
235		- exit;
236		- }
237		- $api_key = $preflight['api_key'];
238		-
239		- // Metadata is fetched in BOTH modes — it is the reliable half of auto, and in
240		- // manual mode it enriches the indexed text with the real title/channel.
241		- $meta = $this->mxchat_fetch_youtube_oembed($video_id);
242		- $video_title = isset($meta['title']) ? sanitize_text_field($meta['title']) : '';
243		- $video_channel = isset($meta['author_name']) ? sanitize_text_field($meta['author_name']) : '';
244		-
245		- $header_lines = 'YouTube Video: ' . ($video_title !== '' ? $video_title : $canonical_url) . "\n";
246		- if ($video_channel !== '') {
247		- $header_lines .= 'Channel: ' . $video_channel . "\n";
248		- }
249		- $header_lines .= 'URL: ' . $canonical_url . "\n\n";
250		-
251		- $transcript_missing = false;
252		-
253		- if ($description_mode === 'manual') {
254		- if ($manual_description === '') {
255		- set_transient('mxchat_admin_notice_error',
256		- esc_html__('Please write a description for the video, or switch to auto-fetch.', 'mxchat'),
257		- 30
258		- );
259		- wp_safe_redirect(esc_url($redirect_url));
260		- exit;
261		- }
262		- $indexed_text = $header_lines . $manual_description;
263		- } else {
264		- $transcript = $this->mxchat_fetch_youtube_transcript($video_id);
265		-
266		- if (strlen($transcript) >= 200) {
267		- $indexed_text = $header_lines . $transcript;
268		- } else {
269		- // Graceful fallback: captions disabled / blocked / no speech. Auto
270		- // reliably gets metadata; it does NOT guarantee a transcript.
271		- $transcript_missing = true;
272		-
273		- if ($video_title === '' && $video_channel === '') {
274		- // Both halves failed — nothing meaningful to index.
275		- set_transient('mxchat_admin_notice_error',
276		- esc_html__('Could not retrieve any information for that video (no metadata and no captions). Please check the URL, or use the manual description option.', 'mxchat'),
277		- 30
278		- );
279		- wp_safe_redirect(esc_url($redirect_url));
280		- exit;
281		- }
282		-
283		- $indexed_text = $header_lines . sprintf(
284		- /* translators: 1: video title, 2: channel name */
285		- __('A YouTube video titled "%1$s" from the channel %2$s.', 'mxchat'),
286		- $video_title !== '' ? $video_title : $canonical_url,
287		- $video_channel !== '' ? $video_channel : 'YouTube'
288		- );
289		- }
290		- }
291		-
292		- $result = MxChat_Utils::submit_content_to_db($indexed_text, $canonical_url, $api_key, null, $bot_id, 'youtube');
293		-
294		- if (is_wp_error($result)) {
295		- set_transient('mxchat_admin_notice_error',
296		- esc_html__('Error storing video in the knowledge base: ', 'mxchat') . $result->get_error_message(),
297		- 30
298		- );
299		- wp_safe_redirect(esc_url($redirect_url));
300		- exit;
301		- }
302		-
303		- if ($transcript_missing) {
304		- set_transient('mxchat_admin_notice_success',
305		- esc_html__('Video indexed from its title and channel — no captions were available for a transcript. The form below is pre-filled: write your own description and import again to improve matching (it updates the same entry).', 'mxchat'),
306		- 30
307		- );
308		- // Bounce back with prefill args so the page reopens the YouTube form in
309		- // manual mode with the URL + fetched title ready to augment.
310		- $redirect_url = add_query_arg(array(
311		- 'mxchat_yt_prefill' => '1',
312		- 'yt_url' => rawurlencode($canonical_url),
313		- 'yt_title' => rawurlencode($video_title),
314		- ), $redirect_url);
315		- } else {
316		- set_transient('mxchat_admin_notice_success',
317		- esc_html__('YouTube video successfully added to the knowledge base!', 'mxchat'),
318		- 30
319		- );
320		- }
321		-
322		- wp_safe_redirect(esc_url_raw($redirect_url));
323		- exit;
324		-}
325		-
326		-/**
327		- * Fetch YouTube oEmbed metadata for a video (no API key required).
328		- * Returns the decoded array (title, author_name, thumbnail_url, ...) or array().
329		- */
330		-private function mxchat_fetch_youtube_oembed($video_id) {
331		- $oembed_url = 'https://www.youtube.com/oembed?url=' . rawurlencode('https://www.youtube.com/watch?v=' . $video_id) . '&format=json';
332		- $response = wp_remote_get($oembed_url, array('timeout' => 15));
333		- if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
334		- return array();
335		- }
336		- $data = json_decode(wp_remote_retrieve_body($response), true);
337		- return is_array($data) ? $data : array();
338		-}
339		-
340		-/**
341		- * Best-effort captions transcript for a video. Deliberately ISOLATED: this uses
342		- * YouTube's unofficial timedtext route (the caption track list embedded in the
343		- * watch page), which YouTube has broken before and will break again. Every
344		- * failure mode returns '' so a break degrades to the metadata-only import path
345		- * instead of erroring the whole submission. Do not let anything in here throw.
346		- */
347		-private function mxchat_fetch_youtube_transcript($video_id) {
348		- $watch_url = 'https://www.youtube.com/watch?v=' . $video_id . '&hl=en';
349		-
350		- // First try the honest ingest UA; some responses omit the player config for
351		- // bot UAs, so retry once with a browser UA before giving up.
352		- $user_agents = array(
353		- mxchat_ingest_user_agent(),
354		- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
355		- );
356		-
357		- $tracks = array();
358		- foreach ($user_agents as $ua) {
359		- $response = wp_remote_get($watch_url, array(
360		- 'timeout' => 20,
361		- 'user-agent' => $ua,
362		- ));
363		- if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
364		- continue;
365		- }
366		- $body = wp_remote_retrieve_body($response);
367		- if (!is_string($body) || $body === '' || !preg_match('/"captionTracks":(\[.*?\])(?=,")/s', $body, $m)) {
368		- continue;
369		- }
370		- $decoded = json_decode($m[1], true);
371		- if (is_array($decoded) && !empty($decoded)) {
372		- $tracks = $decoded;
373		- break;
374		- }
375		- }
376		-
377		- if (empty($tracks)) {
378		- return '';
379		- }
380		-
381		- // Prefer an English track, else take the first offered.
382		- $chosen = null;
383		- foreach ($tracks as $track) {
384		- if (isset($track['languageCode']) && strpos($track['languageCode'], 'en') === 0) {
385		- $chosen = $track;
386		- break;
387		- }
388		- }
389		- if ($chosen === null) {
390		- $chosen = $tracks[0];
391		- }
392		- if (empty($chosen['baseUrl']) || !is_string($chosen['baseUrl'])) {
393		- return '';
394		- }
395		-
396		- $timedtext = wp_remote_get($chosen['baseUrl'], array('timeout' => 20));
397		- if (is_wp_error($timedtext) || wp_remote_retrieve_response_code($timedtext) !== 200) {
398		- return '';
399		- }
400		- $xml = wp_remote_retrieve_body($timedtext);
401		- if (!is_string($xml) || strpos($xml, '<text') === false) {
402		- return '';
403		- }
404		-
405		- // <text start=".." dur="..">caption</text> — strip tags, decode the
406		- // double-encoded entities timedtext ships, collapse whitespace.
407		- $text = preg_replace('/<[^>]+>/', ' ', $xml);
408		- $text = html_entity_decode(html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8'), ENT_QUOTES | ENT_HTML5, 'UTF-8');
409		- $text = trim(preg_replace('/\s+/u', ' ', $text));
410		-
411		- return $text;
412		-}
413		-
414	439	public function mxchat_is_pdf_url($url, $response) {
415	440	$content_type = wp_remote_retrieve_header($response, 'content-type');
416	441	$file_extension = strtolower(pathinfo($url, PATHINFO_EXTENSION));
417	442
418		- // Check Content-Disposition header for .pdf filename (Google Drive sends this)
419		- $disposition = wp_remote_retrieve_header($response, 'content-disposition');
420		- $has_pdf_disposition = ! empty($disposition) && stripos($disposition, '.pdf') !== false;
421		-
422		- return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf' || $has_pdf_disposition;
	443	+ return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf';
423	444	}
424		-
425		-
426		-public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response, $bot_id = 'default') {
	445	+public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response) {
427	446	if (!current_user_can('manage_options')) {
	447	+ //error_log(esc_html__('Unauthorized PDF processing attempt', 'mxchat'));
428	448	return false;
429	449	}
430	450
431	451	$pdf_url = esc_url_raw($pdf_url);
		@@ -431,8 +451,9 @@
431	451	$pdf_url = esc_url_raw($pdf_url);
432	452	$upload_dir = wp_upload_dir();
433	453
434	454	if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
	455	+ //error_log(sprintf(esc_html__('Upload directory error: %s', 'mxchat'), esc_html($upload_dir['error'])));
435	456	return false;
436	457	}
437	458
438	459	$pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
		@@ -439,12 +460,14 @@
439	460	$pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
440	461
441	462	$response_body = wp_remote_retrieve_body($response);
442	463	if (empty($response_body)) {
	464	+ //error_log(esc_html__('Empty PDF response body', 'mxchat'));
443	465	return false;
444	466	}
445	467
446	468	if (!wp_mkdir_p(dirname($pdf_path))) {
	469	+ //error_log(sprintf(esc_html__('Failed to create directory for PDF: %s', 'mxchat'), esc_html($pdf_path)));
447	470	return false;
448	471	}
449	472
450	473	try {
		@@ -453,1140 +476,303 @@
453	476	if (!file_exists($pdf_path)) {
454	477	throw new Exception(__('Failed to save PDF file', 'mxchat'));
455	478	}
456	479
457		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
458		-
459		- if ($total_pages === false || $total_pages < 1) {
460		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
461		- }
	480	+ $parser = new \Smalot\PdfParser\Parser();
	481	+ $pdf = $parser->parseFile($pdf_path);
	482	+ $total_pages = absint(count($pdf->getPages()));
462	483
463		- // Create unique queue ID
464		- $queue_id = 'pdf_' . md5($pdf_url . time());
465		-
466		- // Create array of pages to process
467		- $pages = array();
468		- for ($i = 1; $i <= $total_pages; $i++) {
469		- $pages[] = array(
470		- 'pdf_path' => $pdf_path,
471		- 'pdf_url' => $pdf_url,
472		- 'page_number' => $i,
473		- 'total_pages' => $total_pages
474		- );
	484	+ if ($total_pages < 1) {
	485	+ throw new Exception(__('Invalid PDF: no pages found', 'mxchat'));
475	486	}
476	487
477		- // Add pages to queue
478		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
	488	+ wp_schedule_single_event(time(), 'mxchat_process_pdf_pages', array(
	489	+ 'pdf_path' => $pdf_path,
	490	+ 'pdf_url' => $pdf_url,
	491	+ 'total_pages' => $total_pages,
	492	+ 'batch_size' => absint(15),
	493	+ 'batch_pause' => absint(10)
	494	+ ));
479	495
480		- if ($queued_count === 0) {
481		- wp_delete_file($pdf_path);
482		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
483		- }
	496	+ $status_data = array(
	497	+ 'total_pages' => $total_pages,
	498	+ 'processed_pages' => 0,
	499	+ 'status' => 'processing',
	500	+ 'last_update' => time()
	501	+ );
484	502
485		- // Store queue metadata
486		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $pdf_url);
487		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
488		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
489		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
490		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
491		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	503	+ set_transient(
	504	+ sanitize_key('mxchat_pdf_status_' . md5($pdf_url)),
	505	+ array_map('sanitize_text_field', $status_data),
	506	+ DAY_IN_SECONDS
	507	+ );
492	508
493		- // Store queue ID in transient for status tracking
494		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
495		- set_transient('mxchat_last_pdf_url', $pdf_url, DAY_IN_SECONDS);
	509	+ return __('scheduled', 'mxchat');
496	510
497		- return 'queued';
498		-
499	511	} catch (Exception $e) {
	512	+ //error_log(sprintf(esc_html__('Error preparing PDF for processing: %s', 'mxchat'), esc_html($e->getMessage())));
500	513	if (file_exists($pdf_path)) {
501	514	wp_delete_file($pdf_path);
502	515	}
503		- return $e->getMessage();
	516	+ return false;
504	517	}
505	518	}
506	519
507		-/**
508		- * Handle direct PDF file upload from the knowledge base page
509		- */
510		-public function mxchat_handle_pdf_file_submission() {
511		- if (!isset($_POST['submit_pdf_file']) || !current_user_can('manage_options')) {
512		- wp_die(esc_html__('Unauthorized access', 'mxchat'));
513		- }
	520	+public function mxchat_process_pdf_pages_cron($pdf_path, $pdf_url, $total_pages, $batch_size, $batch_pause) {
	521	+ // Validate inputs
	522	+ $pdf_path = sanitize_text_field($pdf_path);
	523	+ $pdf_url = esc_url_raw($pdf_url);
	524	+ $total_pages = absint($total_pages);
	525	+ $batch_size = absint($batch_size);
	526	+ $batch_pause = absint($batch_pause);
514	527
515		- check_admin_referer('mxchat_submit_pdf_file_action', 'mxchat_submit_pdf_file_nonce');
516		-
517		- $redirect_url = admin_url('admin.php?page=mxchat-prompts');
518		-
519		- // Validate file upload
520		- if (empty($_FILES['pdf_file']) || $_FILES['pdf_file']['error'] !== UPLOAD_ERR_OK) {
521		- $error_code = isset($_FILES['pdf_file']['error']) ? $_FILES['pdf_file']['error'] : UPLOAD_ERR_NO_FILE;
522		- $error_messages = array(
523		- UPLOAD_ERR_INI_SIZE => __('The uploaded file exceeds the server upload_max_filesize limit.', 'mxchat'),
524		- UPLOAD_ERR_FORM_SIZE => __('The uploaded file exceeds the form MAX_FILE_SIZE limit.', 'mxchat'),
525		- UPLOAD_ERR_PARTIAL => __('The file was only partially uploaded.', 'mxchat'),
526		- UPLOAD_ERR_NO_FILE => __('No file was uploaded. Please select a PDF file.', 'mxchat'),
527		- UPLOAD_ERR_NO_TMP_DIR => __('Server missing temporary folder.', 'mxchat'),
528		- UPLOAD_ERR_CANT_WRITE => __('Server failed to write file to disk.', 'mxchat'),
529		- );
530		- $error_msg = isset($error_messages[$error_code]) ? $error_messages[$error_code] : __('Unknown upload error.', 'mxchat');
531		- set_transient('mxchat_admin_notice_error', $error_msg, 30);
532		- wp_safe_redirect(esc_url($redirect_url));
533		- exit;
534		- }
535		-
536		- $file = $_FILES['pdf_file'];
537		-
538		- // Validate MIME type
539		- $finfo = finfo_open(FILEINFO_MIME_TYPE);
540		- $mime_type = finfo_file($finfo, $file['tmp_name']);
541		- finfo_close($finfo);
542		-
543		- if ($mime_type !== 'application/pdf') {
544		- set_transient('mxchat_admin_notice_error',
545		- esc_html__('Invalid file type. Only PDF files are accepted.', 'mxchat'),
546		- 30
547		- );
548		- wp_safe_redirect(esc_url($redirect_url));
549		- exit;
550		- }
551		-
552		- // Validate extension
553		- $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
554		- if ($ext !== 'pdf') {
555		- set_transient('mxchat_admin_notice_error',
556		- esc_html__('Invalid file extension. Only .pdf files are accepted.', 'mxchat'),
557		- 30
558		- );
559		- wp_safe_redirect(esc_url($redirect_url));
560		- exit;
561		- }
562		-
563		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
564		- $original_filename = sanitize_file_name($file['name']);
565		-
566		- $upload_dir = wp_upload_dir();
567		- if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
568		- set_transient('mxchat_admin_notice_error',
569		- esc_html__('WordPress upload directory is not writable.', 'mxchat'),
570		- 30
571		- );
572		- wp_safe_redirect(esc_url($redirect_url));
573		- exit;
574		- }
575		-
576		- $pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
577		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
578		-
579		- if (!wp_mkdir_p(dirname($pdf_path))) {
580		- set_transient('mxchat_admin_notice_error',
581		- esc_html__('Failed to create upload directory.', 'mxchat'),
582		- 30
583		- );
584		- wp_safe_redirect(esc_url($redirect_url));
585		- exit;
586		- }
587		-
588		- // Move uploaded file
589		- if (!move_uploaded_file($file['tmp_name'], $pdf_path)) {
590		- set_transient('mxchat_admin_notice_error',
591		- esc_html__('Failed to save uploaded PDF file.', 'mxchat'),
592		- 30
593		- );
594		- wp_safe_redirect(esc_url($redirect_url));
595		- exit;
596		- }
597		-
598	528	try {
599		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
600		-
601		- if ($total_pages === false || $total_pages < 1) {
602		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
	529	+ if (!file_exists($pdf_path)) {
	530	+ throw new Exception(sprintf('PDF file not found at path: %s', esc_html($pdf_path)));
603	531	}
604	532
605		- // Use original filename as the source identifier
606		- $source_label = 'upload://' . $original_filename;
	533	+ $parser = new \Smalot\PdfParser\Parser();
	534	+ $pdf = $parser->parseFile($pdf_path);
	535	+ $pages = $pdf->getPages();
607	536
608		- $queue_id = 'pdf_' . md5($source_label . time());
	537	+ // Get current progress
	538	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	539	+ $status = get_transient($status_key);
609	540
610		- $pages = array();
611		- for ($i = 1; $i <= $total_pages; $i++) {
612		- $pages[] = array(
613		- 'pdf_path' => $pdf_path,
614		- 'pdf_url' => $source_label,
615		- 'page_number' => $i,
616		- 'total_pages' => $total_pages,
617		- );
	541	+ if (!$status || !is_array($status)) {
	542	+ throw new Exception('Invalid status data retrieved from transient');
618	543	}
619	544
620		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
621		-
622		- if ($queued_count === 0) {
623		- wp_delete_file($pdf_path);
624		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
	545	+ // Initialize failed pages list if it doesn't exist
	546	+ if (!isset($status['failed_pages_list']) || !is_array($status['failed_pages_list'])) {
	547	+ $status['failed_pages_list'] = [];
625	548	}
626	549
627		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $source_label);
628		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
629		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
630		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
631		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
632		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
633		-
634		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
635		- set_transient('mxchat_last_pdf_url', $source_label, DAY_IN_SECONDS);
636		-
637		- set_transient('mxchat_admin_notice_success',
638		- sprintf(
639		- esc_html__('PDF "%s" (%d pages) queued for processing. Processing will start automatically.', 'mxchat'),
640		- esc_html($original_filename),
641		- $total_pages
642		- ),
643		- 30
644		- );
645		-
646		- } catch (Exception $e) {
647		- if (file_exists($pdf_path)) {
648		- wp_delete_file($pdf_path);
	550	+ $start_page = absint($status['processed_pages']);
	551	+ $end_page = min($start_page + $batch_size, $total_pages);
	552	+ $options = get_option('mxchat_options');
	553	+
	554	+ if (empty($options['api_key'])) {
	555	+ throw new Exception('API key is missing or invalid');
649	556	}
650		- set_transient('mxchat_admin_notice_error',
651		- esc_html__('Failed to process uploaded PDF: ', 'mxchat') . esc_html($e->getMessage()),
652		- 30
653		- );
654		- }
655	557
656		- wp_safe_redirect(esc_url($redirect_url));
657		- exit;
658		-}
	558	+ $successful_pages = 0;
	559	+ $failed_pages = 0;
659	560
660		-/**
661		- * Handle direct document upload (.docx / .txt / .md) from the knowledge base
662		- * page (plan 0485e5). Unlike PDF Upload there is no per-page queue: the text
663		- * extracts in one pass and routes through submit_content_to_db, whose chunker
664		- * takes over for long content. The uploaded file is read from the PHP temp
665		- * file and never persisted — only its extracted text enters the KB.
666		- *
667		- * Source identity matches PDF Upload's scheme: upload://<filename>, stable
668		- * across re-uploads so a re-import REPLACES (delete_chunks_for_url + upsert
669		- * per identity) instead of duplicating.
670		- */
671		-public function mxchat_handle_document_file_submission() {
672		- if (!isset($_POST['submit_document_file']) || !current_user_can('manage_options')) {
673		- wp_die(esc_html__('Unauthorized access', 'mxchat'));
674		- }
	561	+ for ($i = $start_page; $i < $end_page; $i++) {
	562	+ $page_number = $i + 1;
	563	+ $max_retries = 3;
	564	+ $retry_count = 0;
	565	+ $page_processed = false;
	566	+ $last_error = '';
675	567
676		- check_admin_referer('mxchat_submit_document_file_action', 'mxchat_submit_document_file_nonce');
	568	+ while (!$page_processed && $retry_count < $max_retries) {
	569	+ try {
	570	+ $text = $pages[$i]->getText();
	571	+
	572	+ if (empty($text)) {
	573	+ throw new Exception("Empty text on page {$page_number}");
	574	+ }
	575	+
	576	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
677	577
678		- $redirect_url = admin_url('admin.php?page=mxchat-prompts');
	578	+ if (empty($sanitized_content)) {
	579	+ throw new Exception("No valid content after sanitization on page {$page_number}");
	580	+ }
679	581
680		- if (empty($_FILES['document_file']) || $_FILES['document_file']['error'] !== UPLOAD_ERR_OK) {
681		- $error_code = isset($_FILES['document_file']['error']) ? $_FILES['document_file']['error'] : UPLOAD_ERR_NO_FILE;
682		- $error_messages = array(
683		- UPLOAD_ERR_INI_SIZE => __('The uploaded file exceeds the server upload_max_filesize limit.', 'mxchat'),
684		- UPLOAD_ERR_FORM_SIZE => __('The uploaded file exceeds the form MAX_FILE_SIZE limit.', 'mxchat'),
685		- UPLOAD_ERR_PARTIAL => __('The file was only partially uploaded.', 'mxchat'),
686		- UPLOAD_ERR_NO_FILE => __('No file was uploaded. Please select a document.', 'mxchat'),
687		- UPLOAD_ERR_NO_TMP_DIR => __('Server missing temporary folder.', 'mxchat'),
688		- UPLOAD_ERR_CANT_WRITE => __('Server failed to write file to disk.', 'mxchat'),
689		- );
690		- $error_msg = isset($error_messages[$error_code]) ? $error_messages[$error_code] : __('Unknown upload error.', 'mxchat');
691		- set_transient('mxchat_admin_notice_error', $error_msg, 30);
692		- wp_safe_redirect(esc_url($redirect_url));
693		- exit;
694		- }
	582	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	583	+
	584	+ if (is_string($embedding_vector)) {
	585	+ throw new Exception("Embedding generation failed: " . $embedding_vector);
	586	+ }
	587	+
	588	+ if (!is_array($embedding_vector)) {
	589	+ throw new Exception("Embedding generation returned unexpected result type: " . gettype($embedding_vector));
	590	+ }
	591	+
	592	+ $metadata = array(
	593	+ 'document_type' => 'pdf',
	594	+ 'total_pages' => $total_pages,
	595	+ 'current_page' => $page_number,
	596	+ 'prev_page' => $i > 0 ? $i : null,
	597	+ 'next_page' => $i < ($total_pages - 1) ? $i + 2 : null,
	598	+ 'source_url' => $pdf_url
	599	+ );
695	600
696		- $file = $_FILES['document_file'];
697		- $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
	601	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	602	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
698	603
699		- $finfo = finfo_open(FILEINFO_MIME_TYPE);
700		- $mime_type = finfo_file($finfo, $file['tmp_name']);
701		- finfo_close($finfo);
	604	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $options['api_key']);
	605	+
	606	+ if (is_wp_error($db_result)) {
	607	+ throw new Exception("Database submission failed: " . $db_result->get_error_message());
	608	+ }
702	609
703		- // Per-extension MIME expectations. finfo commonly reports .docx as
704		- // application/zip (it IS a Zip container) and .md as plain text.
705		- $mime_ok = false;
706		- if ($ext === 'docx') {
707		- $mime_ok = in_array($mime_type, array(
708		- 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
709		- 'application/zip',
710		- ), true);
711		- } elseif ($ext === 'txt' || $ext === 'md') {
712		- $mime_ok = (strpos((string) $mime_type, 'text/') === 0);
713		- }
	610	+ // Success!
	611	+ $page_processed = true;
	612	+ $successful_pages++;
	613	+
	614	+ } catch (Exception $e) {
	615	+ $retry_count++;
	616	+ $last_error = $e->getMessage();
	617	+
	618	+ //error_log("PDF page {$page_number} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	619	+
	620	+ if ($retry_count < $max_retries) {
	621	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	622	+ sleep(pow(2, $retry_count - 1));
	623	+ }
	624	+ }
	625	+ }
714	626
715		- if (!$mime_ok) {
716		- set_transient('mxchat_admin_notice_error',
717		- esc_html__('Invalid or unreadable document. Accepted types: .docx, .txt, .md.', 'mxchat'),
718		- 30
719		- );
720		- wp_safe_redirect(esc_url($redirect_url));
721		- exit;
722		- }
	627	+ // If page still not processed after all retries, mark as failed
	628	+ if (!$page_processed) {
	629	+ $failed_pages++;
	630	+ $status['failed_pages_list'][] = [
	631	+ 'page' => $page_number,
	632	+ 'error' => $last_error,
	633	+ 'time' => time(),
	634	+ 'retries' => $max_retries
	635	+ ];
	636	+
	637	+ // Limit failed pages list to prevent memory issues
	638	+ if (count($status['failed_pages_list']) > 50) {
	639	+ $status['failed_pages_list'] = array_slice($status['failed_pages_list'], -50);
	640	+ }
	641	+
	642	+ //error_log("PDF page {$page_number} permanently failed after {$max_retries} attempts: " . $last_error);
	643	+ }
723	644
724		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
725		- $original_filename = sanitize_file_name($file['name']);
726		-
727		- // ---- Extract text (ONE extractor for .docx — the word handler's) ----
728		- if ($ext === 'docx') {
729		- $text = MXChat_Word_Handler::extract_docx_text($file['tmp_name']);
730		- if ($text === false) {
731		- set_transient('mxchat_admin_notice_error',
732		- esc_html__('The .docx file could not be read. It may be corrupt, empty, or not a real Word document.', 'mxchat'),
733		- 30
734		- );
735		- wp_safe_redirect(esc_url($redirect_url));
736		- exit;
	645	+ // Update progress
	646	+ $status['processed_pages'] = absint($page_number);
	647	+ $status['last_update'] = time();
	648	+ $status['failed_pages'] = absint($status['failed_pages'] ?? 0) + ($page_processed ? 0 : 1);
	649	+
	650	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
737	651	}
738		- } else {
739		- // .txt / .md read as-is. Markdown keeps its syntax on purpose —
740		- // headings are useful retrieval signal.
741		- $text = (string) file_get_contents($file['tmp_name']);
742		- $text = wp_check_invalid_utf8($text);
743		- $text = trim($text);
744		- }
745	652
746		- if ($text === '') {
747		- set_transient('mxchat_admin_notice_error',
748		- esc_html__('The uploaded document contains no readable text.', 'mxchat'),
749		- 30
750		- );
751		- wp_safe_redirect(esc_url($redirect_url));
752		- exit;
753		- }
754		-
755		- // Size cap — same pdf_max_pages setting the PDF/toolbar paths use, but
756		- // estimated by CHARACTERS (~2500/page): the .docx cleaner collapses all
757		- // newlines to spaces, so a paragraph count reads 1 for any Word file.
758		- // Processing is synchronous — an unbounded document risks a timeout.
759		- $options = get_option('mxchat_options', array());
760		- $max_pages = isset($options['pdf_max_pages']) ? intval($options['pdf_max_pages']) : 69;
761		- $estimated_pages = (int) ceil(strlen($text) / 2500);
762		- if ($estimated_pages > $max_pages) {
763		- set_transient('mxchat_admin_notice_error',
764		- sprintf(
765		- esc_html__('The document is too large (about %1$d pages; the limit is %2$d). Split it into smaller files, or raise the PDF max pages setting.', 'mxchat'),
766		- $estimated_pages,
767		- $max_pages
768		- ),
769		- 30
770		- );
771		- wp_safe_redirect(esc_url($redirect_url));
772		- exit;
773		- }
774		-
775		- // Embedding API key — bot-aware, same shape as the direct-content handler.
776		- $api_key = '';
777		- if ($bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager')) {
778		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
779		- $api_key = $bot_options['api_key'] ?? '';
780		- }
781		- if (empty($api_key)) {
782		- $api_key = $options['api_key'] ?? '';
783		- }
784		-
785		- // Stable identity — PDF Upload's scheme. A re-upload of the same filename
786		- // replaces: clear old chunks first (covers a doc shrinking below the chunk
787		- // threshold, where the single-vector path would not clean them), then
788		- // submit — the chunked path re-deletes harmlessly.
789		- $source_label = 'upload://' . $original_filename;
790		- MxChat_Utils::delete_chunks_for_url($source_label, $bot_id);
791		- $result = MxChat_Utils::submit_content_to_db($text, $source_label, $api_key, null, $bot_id, 'document');
792		-
793		- if (is_wp_error($result)) {
794		- set_transient('mxchat_admin_notice_error',
795		- esc_html__('Failed to import the document: ', 'mxchat') . esc_html($result->get_error_message()),
796		- 30
797		- );
798		- } else {
799		- set_transient('mxchat_admin_notice_success',
800		- sprintf(
801		- esc_html__('Document "%s" imported into the knowledge base.', 'mxchat'),
802		- esc_html($original_filename)
803		- ),
804		- 30
805		- );
806		- }
807		-
808		- wp_safe_redirect(esc_url($redirect_url));
809		- exit;
810		-}
811		-
812		-/**
813		- * Validate PDF and count pages with multiple parser attempts
814		- */
815		-private function mxchat_validate_and_count_pdf_pages($pdf_path) {
816		- // Method 1: Try with Smalot PDF Parser (your current method)
817		- try {
818		- mxchat_load_pdf_parser();
819		- $parser = new \Smalot\PdfParser\Parser();
820		- $pdf = $parser->parseFile($pdf_path);
821		- $pages = $pdf->getPages();
822		- $page_count = count($pages);
823		-
824		- if ($page_count > 0) {
825		- //error_log('PDF parsed successfully with Smalot parser: ' . $page_count . ' pages');
826		- return $page_count;
827		- }
828		- } catch (Exception $e) {
829		- //error_log('Smalot PDF parser failed: ' . $e->getMessage());
830		- }
831		-
832		- // Method 2: Try with pdfinfo command (if available)
833		- if (function_exists('shell_exec') && !$this->mxchat_is_shell_disabled()) {
834		- try {
835		- $command = 'pdfinfo ' . escapeshellarg($pdf_path) . ' 2>&1';
836		- $output = shell_exec($command);
	653	+ // Schedule next batch if needed
	654	+ if ($end_page < $total_pages) {
	655	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_pdf_pages', array(
	656	+ 'pdf_path' => $pdf_path,
	657	+ 'pdf_url' => $pdf_url,
	658	+ 'total_pages' => $total_pages,
	659	+ 'batch_size' => $batch_size,
	660	+ 'batch_pause' => $batch_pause
	661	+ ));
	662	+ } else {
	663	+ // Processing complete
	664	+ $status['status'] = 'complete';
	665	+ $status['processed_pages'] = $total_pages;
837	666
838		- if ($output && preg_match('/Pages:\s*(\d+)/', $output, $matches)) {
839		- $page_count = intval($matches[1]);
840		- if ($page_count > 0) {
841		- //error_log('PDF parsed successfully with pdfinfo: ' . $page_count . ' pages');
842		- return $page_count;
843		- }
844		- }
845		- } catch (Exception $e) {
846		- //error_log('pdfinfo command failed: ' . $e->getMessage());
847		- }
848		- }
849		-
850		- // Method 3: Try to repair PDF and parse again
851		- try {
852		- $repaired_path = $this->mxchat_attempt_pdf_repair($pdf_path);
853		- if ($repaired_path && $repaired_path !== $pdf_path) {
854		- mxchat_load_pdf_parser();
855		- $parser = new \Smalot\PdfParser\Parser();
856		- $pdf = $parser->parseFile($repaired_path);
857		- $pages = $pdf->getPages();
858		- $page_count = count($pages);
	667	+ // Add completion summary
	668	+ $status['completion_summary'] = [
	669	+ 'total_pages' => $total_pages,
	670	+ 'successful_pages' => $total_pages - absint($status['failed_pages'] ?? 0),
	671	+ 'failed_pages' => absint($status['failed_pages'] ?? 0),
	672	+ 'completion_time' => current_time('mysql')
	673	+ ];
859	674
860		- if ($page_count > 0) {
861		- // Replace original with repaired version
862		- copy($repaired_path, $pdf_path);
863		- unlink($repaired_path);
864		- //error_log('PDF repaired and parsed successfully: ' . $page_count . ' pages');
865		- return $page_count;
	675	+ // Save the completed status (don't delete it - let user dismiss manually)
	676	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	677	+
	678	+ // Clean up the temporary PDF file
	679	+ if (file_exists($pdf_path)) {
	680	+ wp_delete_file($pdf_path);
866	681	}
867	682
868		- // Clean up repaired file if it didn't work
869		- unlink($repaired_path);
	683	+ // DON'T delete the status transients here - let user dismiss manually
870	684	}
871		- } catch (Exception $e) {
872		- //error_log('PDF repair attempt failed: ' . $e->getMessage());
873		- }
874	685
875		- // Method 4: Manual PDF structure analysis (basic page count)
876		- try {
877		- $page_count = $this->mxchat_manual_pdf_page_count($pdf_path);
878		- if ($page_count > 0) {
879		- //error_log('PDF page count determined manually: ' . $page_count . ' pages');
880		- return $page_count;
	686	+ } catch (\Exception $e) {
	687	+ //error_log(sprintf('[MXCHAT-PDF] Error processing PDF: %s', $e->getMessage()));
	688	+
	689	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	690	+ $status = get_transient($status_key);
	691	+
	692	+ if (!$status || !is_array($status)) {
	693	+ $status = array(
	694	+ 'total_pages' => $total_pages,
	695	+ 'processed_pages' => 0,
	696	+ 'status' => 'error',
	697	+ 'error' => sanitize_text_field($e->getMessage()),
	698	+ 'last_update' => time()
	699	+ );
	700	+ } else {
	701	+ $status['status'] = 'error';
	702	+ $status['error'] = sanitize_text_field($e->getMessage());
	703	+ $status['last_update'] = time();
881	704	}
882		- } catch (Exception $e) {
883		- //error_log('Manual PDF analysis failed: ' . $e->getMessage());
884		- }
885		-
886		- //error_log('All PDF parsing methods failed for: ' . $pdf_path);
887		- return false;
888		-}
889		-
890		-/**
891		- * Check if shell_exec is disabled
892		- */
893		-private function mxchat_is_shell_disabled() {
894		- $disabled = explode(',', ini_get('disable_functions'));
895		- return in_array('shell_exec', $disabled);
896		-}
897		-
898		-/**
899		- * Attempt to repair PDF using basic methods
900		- */
901		-private function mxchat_attempt_pdf_repair($pdf_path) {
902		- try {
903		- $content = file_get_contents($pdf_path);
904		- if (!$content) {
905		- return false;
	705	+
	706	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	707	+
	708	+ if (file_exists($pdf_path)) {
	709	+ wp_delete_file($pdf_path);
906	710	}
907		-
908		- // Check if PDF starts with proper header
909		- if (substr($content, 0, 4) !== '%PDF') {
910		- // Try to find PDF header in the content
911		- $header_pos = strpos($content, '%PDF');
912		- if ($header_pos !== false && $header_pos < 1024) {
913		- // Remove junk before PDF header
914		- $content = substr($content, $header_pos);
915		- $repaired_path = $pdf_path . '.repaired';
916		- file_put_contents($repaired_path, $content);
917		- return $repaired_path;
918		- }
919		- }
920		-
921		- // Check for EOF marker
922		- $content = rtrim($content);
923		- if (!preg_match('/%%EOF\s*$/', $content)) {
924		- // Add EOF marker if missing
925		- $content .= "\n%%EOF";
926		- $repaired_path = $pdf_path . '.repaired';
927		- file_put_contents($repaired_path, $content);
928		- return $repaired_path;
929		- }
930		-
931		- } catch (Exception $e) {
932		- //error_log('PDF repair error: ' . $e->getMessage());
933	711	}
934		-
935		- return false;
936	712	}
937	713
938		-/**
939		- * Manual PDF page counting by analyzing PDF structure
940		- */
941		-private function mxchat_manual_pdf_page_count($pdf_path) {
942		- try {
943		- $content = file_get_contents($pdf_path);
944		- if (!$content) {
945		- return 0;
946		- }
947		-
948		- // Method 1: Count /Type /Page objects
949		- $page_count = preg_match_all('/\/Type\s*\/Page[^s]/', $content);
950		- if ($page_count > 0) {
951		- return $page_count;
952		- }
953		-
954		- // Method 2: Look for /Count in pages object
955		- if (preg_match('/\/Type\s*\/Pages.*?\/Count\s+(\d+)/', $content, $matches)) {
956		- return intval($matches[1]);
957		- }
958		-
959		- // Method 3: Count page references
960		- $page_count = preg_match_all('/\d+\s+0\s+obj\s*<<[^>]*\/Type\s*\/Page/', $content);
961		- if ($page_count > 0) {
962		- return $page_count;
963		- }
964		-
965		- } catch (Exception $e) {
966		- //error_log('Manual PDF analysis error: ' . $e->getMessage());
967		- }
968		-
969		- return 0;
970		-}
971		-
972		-
973	714	public function mxchat_save_inline_prompt() {
974	715	// DEBUG: Log what we're receiving
975		- //error_log('=== MXCHAT DEBUG ===');
976		- //error_log('POST data: ' . print_r($_POST, true));
977		- //error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
	716	+ error_log('=== MXCHAT DEBUG ===');
	717	+ error_log('POST data: ' . print_r($_POST, true));
	718	+ error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
978	719
979	720	// Check for nonce security
980	721	check_ajax_referer('mxchat_save_inline_nonce', '_ajax_nonce');
981	722
982	723	// If we get here, nonce passed
983		- //error_log('Nonce verification PASSED');
984		-
985		- // Verify permissions
986		- if (!current_user_can('manage_options')) {
987		- wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
988		- return;
989		- }
990		-
991		- global $wpdb;
992		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
993		-
994		- // Validate and sanitize input data
995		- $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
996		- $article_content = isset($_POST['article_content']) ? wp_kses_post(wp_unslash($_POST['article_content'])) : '';
997		- $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
998		-
999		- if ($prompt_id > 0 && !empty($article_content)) {
1000		- // Re-generate the embedding vector for the updated content
1001		- $embedding_vector = $this->mxchat_generate_embedding($article_content);
1002		- if (is_array($embedding_vector)) {
1003		- // Serialize the embedding vector before storing it
1004		- $embedding_vector_serialized = serialize($embedding_vector);
1005		- // Update the prompt in the database
1006		- $updated = $wpdb->update(
1007		- $table_name,
1008		- array(
1009		- 'article_content' => $article_content,
1010		- 'embedding_vector' => $embedding_vector_serialized,
1011		- 'source_url' => $article_url,
1012		- ),
1013		- array('id' => $prompt_id),
1014		- array('%s', '%s', '%s'),
1015		- array('%d')
1016		- );
1017		- if ($updated !== false) {
1018		- wp_send_json_success();
1019		- } else {
1020		- MxChat_Admin::mxchat_log_debug('knowledge_error', 'Database update failed when saving knowledge entry');
1021		- wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
1022		- }
1023		- } else {
1024		- MxChat_Admin::mxchat_log_debug('embedding_error', 'Embedding generation failed for knowledge entry');
1025		- wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
1026		- }
1027		- } else {
1028		- wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
1029		- }
1030		-}
	724	+ error_log('Nonce verification PASSED');
1031	725
	726	+ // Verify permissions
	727	+ if (!current_user_can('manage_options')) {
	728	+ wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
	729	+ return;
	730	+ }
1032	731
1033		-/**
1034		- * AJAX: Get full content for editing — reassembles chunks if needed.
1035		- * Works for both WordPress DB and Pinecone entries.
1036		- */
1037		-/**
1038		- * Sanitize a knowledge entry's source_url from an AJAX request WITHOUT destroying
1039		- * its identity. sanitize_text_field() strips percent-encoded octets (%20, %D7%A9…),
1040		- * so a percent-encoded URL — every non-ASCII permalink — would md5 to a DIFFERENT
1041		- * id than the one it was stored under: reads miss the entry and saves write an
1042		- * orphan copy while the original keeps its stale text. URLs get esc_url_raw
1043		- * (identity-preserving, matches what import stored); non-URL keys (mxchat://,
1044		- * _ungrouped_) keep the old sanitizer.
1045		- */
1046		-private function sanitize_entry_source_url( $raw ) {
1047		- $raw = trim( (string) $raw );
1048		- if ( preg_match( '#^https?://#i', $raw ) ) {
1049		- return esc_url_raw( $raw );
1050		- }
1051		- return sanitize_text_field( $raw );
1052		-}
	732	+ global $wpdb;
	733	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1053	734
1054		-public function ajax_mxchat_get_entry_content() {
1055		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
	735	+ // Validate and sanitize input data
	736	+ $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
	737	+ $article_content = isset($_POST['article_content']) ? sanitize_textarea_field($_POST['article_content']) : '';
	738	+ $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
1056	739
1057		- if ( ! current_user_can('manage_options') ) {
1058		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
1059		- }
	740	+ if ($prompt_id > 0 && !empty($article_content)) {
	741	+ // Re-generate the embedding vector for the updated content
	742	+ $embedding_vector = $this->mxchat_generate_embedding($article_content);
1060	743
1061		- $source_url = $this->sanitize_entry_source_url( isset($_POST['source_url']) ? wp_unslash($_POST['source_url']) : '' );
1062		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
1063		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
1064		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
	744	+ if (is_array($embedding_vector)) {
	745	+ // Serialize the embedding vector before storing it
	746	+ $embedding_vector_serialized = serialize($embedding_vector);
1065	747
1066		- if ( $data_source === 'pinecone' ) {
1067		- // Pinecone ids are strings (md5 hashes, manual_* ids) — absint() would
1068		- // destroy them, so re-read the raw value for this branch only.
1069		- $vector_id = isset($_POST['entry_id']) ? sanitize_text_field( wp_unslash($_POST['entry_id']) ) : '';
1070		- // Pinecone: fetch vectors by source_url, reassemble chunks
1071		- $content = $this->get_pinecone_entry_content( $source_url, $vector_id, $bot_id );
1072		- } else {
1073		- // WordPress DB
1074		- $content = $this->get_wordpress_entry_content( $source_url, $entry_id );
1075		- }
	748	+ // Update the prompt in the database
	749	+ $updated = $wpdb->update(
	750	+ $table_name,
	751	+ array(
	752	+ 'article_content' => $article_content,
	753	+ 'embedding_vector' => $embedding_vector_serialized,
	754	+ 'source_url' => $article_url,
	755	+ ),
	756	+ array('id' => $prompt_id),
	757	+ array('%s', '%s', '%s'),
	758	+ array('%d')
	759	+ );
1076	760
1077		- if ( is_wp_error( $content ) ) {
1078		- wp_send_json_error( array( 'message' => $content->get_error_message() ) );
1079		- }
	761	+ if ($updated !== false) {
	762	+ wp_send_json_success();
	763	+ } else {
	764	+ wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
	765	+ }
	766	+ } else {
	767	+ wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
	768	+ }
	769	+ } else {
	770	+ wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
	771	+ }
	772	+ }
1080	773
1081		- wp_send_json_success( $content );
1082		-}
1083	774
1084		-/**
1085		- * Get content from WordPress DB — reassembles chunks by source_url.
1086		- */
1087		-private function get_wordpress_entry_content( $source_url, $entry_id ) {
1088		- global $wpdb;
1089		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
1090		-
1091		- // If we have a source_url, check for chunks
1092		- if ( ! empty( $source_url ) && strpos( $source_url, 'mxchat://' ) !== 0 ) {
1093		- $rows = $wpdb->get_results( $wpdb->prepare(
1094		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
1095		- $source_url
1096		- ) );
1097		-
1098		- if ( $rows && count( $rows ) > 1 ) {
1099		- // Multiple rows = chunked. Reassemble.
1100		- $chunks = array();
1101		- foreach ( $rows as $row ) {
1102		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
1103		- $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
1104		- $chunks[ $index ] = $parsed['text'];
1105		- }
1106		- ksort( $chunks );
1107		- return array(
1108		- 'content' => implode( "\n\n", $chunks ),
1109		- 'source_url' => $source_url,
1110		- 'is_chunked' => true,
1111		- 'chunk_count' => count( $chunks ),
1112		- 'content_type' => $rows[0]->content_type,
1113		- );
1114		- } elseif ( $rows && count( $rows ) === 1 ) {
1115		- $parsed = MxChat_Chunker::parse_stored_chunk( $rows[0]->article_content );
1116		- return array(
1117		- 'content' => $parsed['text'],
1118		- 'source_url' => $source_url,
1119		- 'entry_id' => $rows[0]->id,
1120		- 'is_chunked' => false,
1121		- 'content_type' => $rows[0]->content_type,
1122		- );
1123		- }
1124		- }
1125		-
1126		- // Fallback: fetch by ID
1127		- if ( $entry_id > 0 ) {
1128		- $row = $wpdb->get_row( $wpdb->prepare(
1129		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
1130		- $entry_id
1131		- ) );
1132		- if ( $row ) {
1133		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
1134		- return array(
1135		- 'content' => $parsed['text'],
1136		- 'source_url' => $row->source_url,
1137		- 'entry_id' => $row->id,
1138		- 'is_chunked' => false,
1139		- 'content_type' => $row->content_type,
1140		- );
1141		- }
1142		- }
1143		-
1144		- return new WP_Error( 'not_found', 'Entry not found.' );
1145		-}
1146		-
1147		-/**
1148		- * Get content from Pinecone — fetches vectors by source_url, reassembles chunks.
1149		- */
1150		-private function get_pinecone_entry_content( $source_url, $entry_id, $bot_id ) {
1151		- if ( ! class_exists('MxChat_Pinecone_Manager') ) {
1152		- return new WP_Error( 'pinecone_unavailable', 'Pinecone manager not available.' );
1153		- }
1154		-
1155		- // Get Pinecone config
1156		- if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
1157		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
1158		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1159		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1160		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1161		- } else {
1162		- $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1163		- $api_key = $bot_config['api_key'] ?? '';
1164		- $host = $bot_config['host'] ?? '';
1165		- $namespace = $bot_config['namespace'] ?? '';
1166		- }
1167		-
1168		- if ( empty($host) || empty($api_key) ) {
1169		- return new WP_Error( 'pinecone_config', 'Pinecone not configured.' );
1170		- }
1171		-
1172		- // Manual entries carry no source_url (their vector id is a minted manual_* string,
1173		- // not md5 of anything the row can hand us) — fetch the exact vector instead.
1174		- // '_ungrouped_' is the table view's synthetic display key for such rows.
1175		- if ( ( empty($source_url) || strpos($source_url, '_ungrouped_') === 0 ) && ! empty($entry_id) && is_string($entry_id) ) {
1176		- $vector_ids = array( $entry_id );
1177		- } else {
1178		- // List vectors with the source_url prefix
1179		- $base_id = md5( $source_url );
1180		- $vector_ids = array( $base_id );
1181		-
1182		- // Find chunk vectors
1183		- // NOTE: Pinecone's /vectors/list is a GET endpoint with query parameters.
1184		- // A POST is answered 200-with-an-empty-body, which reads as "no vectors".
1185		- $list_url = "https://{$host}/vectors/list";
1186		- $list_params = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
1187		- if ( ! empty($namespace) ) {
1188		- $list_params['namespace'] = $namespace;
1189		- }
1190		-
1191		- $list_resp = wp_remote_get( $list_url . '?' . http_build_query( $list_params ), array(
1192		- 'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ),
1193		- 'timeout' => 15,
1194		- ) );
1195		-
1196		- if ( ! is_wp_error($list_resp) ) {
1197		- $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
1198		- if ( ! empty($list_data['vectors']) ) {
1199		- foreach ( $list_data['vectors'] as $v ) {
1200		- $vector_ids[] = $v['id'];
1201		- }
1202		- }
1203		- }
1204		- }
1205		-
1206		- // Fetch vectors with metadata
1207		- // NOTE: /vectors/fetch is a GET endpoint too, and Pinecone expects the ids
1208		- // repeated (ids=a&ids=b) — http_build_query would emit ids[0]=a, so build
1209		- // the query string explicitly.
1210		- $fetch_query = array();
1211		- foreach ( $vector_ids as $fetch_vid ) {
1212		- $fetch_query[] = 'ids=' . rawurlencode( $fetch_vid );
1213		- }
1214		- if ( ! empty($namespace) ) {
1215		- $fetch_query[] = 'namespace=' . rawurlencode( $namespace );
1216		- }
1217		-
1218		- $fetch_resp = wp_remote_get( "https://{$host}/vectors/fetch?" . implode( '&', $fetch_query ), array(
1219		- 'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ),
1220		- 'timeout' => 15,
1221		- ) );
1222		-
1223		- if ( is_wp_error($fetch_resp) ) {
1224		- return new WP_Error( 'pinecone_fetch', 'Failed to fetch from Pinecone.' );
1225		- }
1226		-
1227		- $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
1228		- $vectors = $fetch_data['vectors'] ?? array();
1229		-
1230		- if ( empty($vectors) ) {
1231		- return new WP_Error( 'not_found', 'Entry not found in Pinecone.' );
1232		- }
1233		-
1234		- // Reassemble chunks
1235		- $chunks = array();
1236		- $content_type = 'content';
1237		- foreach ( $vectors as $vid => $vector ) {
1238		- $meta = $vector['metadata'] ?? array();
1239		- $text = $meta['text'] ?? '';
1240		- $index = $meta['chunk_index'] ?? 0;
1241		- $content_type = $meta['type'] ?? 'content';
1242		- $chunks[ intval($index) ] = $text;
1243		- }
1244		- ksort( $chunks );
1245		-
1246		- return array(
1247		- 'content' => implode( "\n\n", $chunks ),
1248		- 'source_url' => $source_url,
1249		- 'is_chunked' => count($chunks) > 1,
1250		- 'chunk_count' => count($chunks),
1251		- 'content_type' => $content_type,
1252		- );
1253		-}
1254		-
1255		-/**
1256		- * AJAX: Inspect a knowledge entry — returns the per-chunk stored text + metadata
1257		- * WITHOUT collapsing it, so a site owner can see exactly what was indexed for an
1258		- * entry (plan-mxchat-20260628-d8cb4b). READ-ONLY: never re-embeds or mutates.
1259		- */
1260		-public function ajax_mxchat_inspect_entry() {
1261		- check_ajax_referer('mxchat_inspect_entry_nonce', 'nonce');
1262		-
1263		- if ( ! current_user_can('manage_options') ) {
1264		- wp_send_json_error( array( 'message' => esc_html__('Permission denied.', 'mxchat') ) );
1265		- }
1266		-
1267		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
1268		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
1269		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
1270		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1271		-
1272		- if ( $data_source === 'pinecone' ) {
1273		- $result = $this->inspect_pinecone_entry( $source_url, $entry_id, $bot_id );
1274		- } else {
1275		- $result = $this->inspect_wordpress_entry( $source_url, $entry_id );
1276		- }
1277		-
1278		- if ( is_wp_error( $result ) ) {
1279		- wp_send_json_error( array( 'message' => $result->get_error_message() ) );
1280		- }
1281		-
1282		- wp_send_json_success( $result );
1283		-}
1284		-
1285		-/**
1286		- * Read-only inspector for WordPress-DB entries. Mirrors get_wordpress_entry_content()
1287		- * but returns each STORED chunk's exact text + length (no implode), plus the assembled
1288		- * embedded text. This shows what is actually in the index, not a re-derivation from the post.
1289		- */
1290		-private function inspect_wordpress_entry( $source_url, $entry_id ) {
1291		- global $wpdb;
1292		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
1293		-
1294		- $rows = array();
1295		-
1296		- // Group by the real stored source_url — this INCLUDES "mxchat://" manual
1297		- // Direct Content entries (the spec's manual-entry case), which share one
1298		- // source_url across their chunk rows. Only the synthetic "_ungrouped_<id>"
1299		- // display key (invented by the table view for rows with no source_url) is
1300		- // excluded; those fall through to the entry_id lookup below.
1301		- if ( ! empty( $source_url ) && strpos( $source_url, '_ungrouped_' ) !== 0 ) {
1302		- $rows = $wpdb->get_results( $wpdb->prepare(
1303		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
1304		- $source_url
1305		- ) );
1306		- }
1307		-
1308		- // Fallback / manual "Direct Content" entries: fetch the single row by id.
1309		- if ( empty( $rows ) && $entry_id > 0 ) {
1310		- $row = $wpdb->get_row( $wpdb->prepare(
1311		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
1312		- $entry_id
1313		- ) );
1314		- if ( $row ) {
1315		- $rows = array( $row );
1316		- }
1317		- }
1318		-
1319		- if ( empty( $rows ) ) {
1320		- return new WP_Error( 'not_found', esc_html__('Entry not found in the local knowledge database.', 'mxchat') );
1321		- }
1322		-
1323		- $chunks = array();
1324		- $content_type = '';
1325		- foreach ( $rows as $row ) {
1326		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
1327		- $text = isset( $parsed['text'] ) ? $parsed['text'] : '';
1328		- $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
1329		- $content_type = $row->content_type;
1330		- $chunks[] = array(
1331		- 'index' => $index,
1332		- 'text' => $text,
1333		- 'length' => function_exists('mb_strlen') ? mb_strlen( $text ) : strlen( $text ),
1334		- 'row_id' => intval( $row->id ),
1335		- );
1336		- }
1337		-
1338		- usort( $chunks, function( $a, $b ) { return $a['index'] - $b['index']; } );
1339		-
1340		- $assembled = implode( "\n\n", wp_list_pluck( $chunks, 'text' ) );
1341		-
1342		- return array(
1343		- 'store' => 'wordpress',
1344		- 'source_url' => $source_url,
1345		- 'content_type' => $content_type,
1346		- 'is_chunked' => count( $chunks ) > 1,
1347		- 'chunk_count' => count( $chunks ),
1348		- 'assembled' => $assembled,
1349		- 'assembled_length' => function_exists('mb_strlen') ? mb_strlen( $assembled ) : strlen( $assembled ),
1350		- 'chunks' => array_values( $chunks ),
1351		- // WP-DB storage carries no separate vector metadata; surface that fact
1352		- // rather than letting the owner guess (the spec's taxonomy question).
1353		- 'metadata' => array(),
1354		- 'metadata_note' => esc_html__('Stored in the local WordPress database. Only the assembled text shown here is embedded — there are no separate vector metadata fields (e.g. taxonomy terms are not stored unless they were injected into the text itself).', 'mxchat'),
1355		- );
1356		-}
1357		-
1358		-/**
1359		- * Read-only inspector for Pinecone entries. Mirrors get_pinecone_entry_content()
1360		- * but keeps each vector's text + metadata instead of imploding, so the owner can
1361		- * confirm exactly which metadata fields (text/source_url/type/last_updated/created_at/bot_id)
1362		- * are present per chunk. READ-ONLY.
1363		- */
1364		-private function inspect_pinecone_entry( $source_url, $entry_id, $bot_id ) {
1365		- if ( ! class_exists('MxChat_Pinecone_Manager') ) {
1366		- return new WP_Error( 'pinecone_unavailable', esc_html__('Pinecone manager not available.', 'mxchat') );
1367		- }
1368		-
1369		- if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
1370		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
1371		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1372		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1373		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1374		- } else {
1375		- $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1376		- $api_key = $bot_config['api_key'] ?? '';
1377		- $host = $bot_config['host'] ?? '';
1378		- $namespace = $bot_config['namespace'] ?? '';
1379		- }
1380		-
1381		- if ( empty($host) || empty($api_key) ) {
1382		- return new WP_Error( 'pinecone_config', esc_html__('Pinecone not configured.', 'mxchat') );
1383		- }
1384		-
1385		- $base_id = md5( $source_url );
1386		- $vector_ids = array( $base_id );
1387		-
1388		- // NOTE: Pinecone's /vectors/list is a GET endpoint with query parameters.
1389		- // A POST is answered 200-with-an-empty-body, which reads as "no vectors".
1390		- $list_url = "https://{$host}/vectors/list";
1391		- $list_params = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
1392		- if ( ! empty($namespace) ) {
1393		- $list_params['namespace'] = $namespace;
1394		- }
1395		-
1396		- $list_resp = wp_remote_get( $list_url . '?' . http_build_query( $list_params ), array(
1397		- 'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ),
1398		- 'timeout' => 15,
1399		- ) );
1400		-
1401		- if ( ! is_wp_error($list_resp) ) {
1402		- $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
1403		- if ( ! empty($list_data['vectors']) ) {
1404		- foreach ( $list_data['vectors'] as $v ) {
1405		- $vector_ids[] = $v['id'];
1406		- }
1407		- }
1408		- }
1409		-
1410		- // NOTE: /vectors/fetch is a GET endpoint too, and Pinecone expects the ids
1411		- // repeated (ids=a&ids=b) — http_build_query would emit ids[0]=a, so build
1412		- // the query string explicitly.
1413		- $fetch_query = array();
1414		- foreach ( $vector_ids as $fetch_vid ) {
1415		- $fetch_query[] = 'ids=' . rawurlencode( $fetch_vid );
1416		- }
1417		- if ( ! empty($namespace) ) {
1418		- $fetch_query[] = 'namespace=' . rawurlencode( $namespace );
1419		- }
1420		-
1421		- $fetch_resp = wp_remote_get( "https://{$host}/vectors/fetch?" . implode( '&', $fetch_query ), array(
1422		- 'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ),
1423		- 'timeout' => 15,
1424		- ) );
1425		-
1426		- if ( is_wp_error($fetch_resp) ) {
1427		- return new WP_Error( 'pinecone_fetch', esc_html__('Failed to fetch from Pinecone.', 'mxchat') );
1428		- }
1429		-
1430		- $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
1431		- $vectors = $fetch_data['vectors'] ?? array();
1432		-
1433		- if ( empty($vectors) ) {
1434		- return new WP_Error( 'not_found', esc_html__('Entry not found in Pinecone.', 'mxchat') );
1435		- }
1436		-
1437		- // Whitelisted metadata fields the spec calls out — shown so devs can confirm
1438		- // what is (and is NOT) stored per vector.
1439		- $meta_fields = array( 'text', 'source_url', 'type', 'last_updated', 'created_at', 'bot_id', 'chunk_index', 'total_chunks' );
1440		- $chunks = array();
1441		- $content_type = '';
1442		- foreach ( $vectors as $vid => $vector ) {
1443		- $meta = isset($vector['metadata']) && is_array($vector['metadata']) ? $vector['metadata'] : array();
1444		- $text = $meta['text'] ?? '';
1445		- $index = isset($meta['chunk_index']) ? intval($meta['chunk_index']) : count($chunks);
1446		- $content_type = $meta['type'] ?? $content_type;
1447		-
1448		- $clean_meta = array();
1449		- foreach ( $meta_fields as $field ) {
1450		- if ( array_key_exists( $field, $meta ) && $field !== 'text' ) {
1451		- $clean_meta[ $field ] = is_scalar( $meta[ $field ] ) ? (string) $meta[ $field ] : wp_json_encode( $meta[ $field ] );
1452		- }
1453		- }
1454		-
1455		- $chunks[] = array(
1456		- 'index' => $index,
1457		- 'text' => $text,
1458		- 'length' => function_exists('mb_strlen') ? mb_strlen( $text ) : strlen( $text ),
1459		- 'vector_id' => (string) $vid,
1460		- 'metadata' => $clean_meta,
1461		- );
1462		- }
1463		-
1464		- usort( $chunks, function( $a, $b ) { return $a['index'] - $b['index']; } );
1465		-
1466		- $assembled = implode( "\n\n", wp_list_pluck( $chunks, 'text' ) );
1467		-
1468		- return array(
1469		- 'store' => 'pinecone',
1470		- 'source_url' => $source_url,
1471		- 'content_type' => $content_type,
1472		- 'is_chunked' => count( $chunks ) > 1,
1473		- 'chunk_count' => count( $chunks ),
1474		- 'assembled' => $assembled,
1475		- 'assembled_length' => function_exists('mb_strlen') ? mb_strlen( $assembled ) : strlen( $assembled ),
1476		- 'chunks' => array_values( $chunks ),
1477		- 'metadata' => array(),
1478		- 'metadata_note' => esc_html__('Stored in Pinecone. Each chunk above lists the vector metadata fields actually present — if a field you expect (such as taxonomy terms) is missing here, it was not stored as metadata and is only searchable if it appears in the embedded text.', 'mxchat'),
1479		- );
1480		-}
1481		-
1482		-/**
1483		- * AJAX: Save edited content — re-chunks and re-embeds as needed.
1484		- * Works for both WordPress DB and Pinecone entries.
1485		- */
1486		-public function ajax_mxchat_save_entry_content() {
1487		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
1488		-
1489		- if ( ! current_user_can('manage_options') ) {
1490		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
1491		- }
1492		-
1493		- $source_url = $this->sanitize_entry_source_url( isset($_POST['source_url']) ? wp_unslash($_POST['source_url']) : '' );
1494		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
1495		- $content = isset($_POST['content']) ? wp_kses_post( wp_unslash($_POST['content']) ) : '';
1496		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
1497		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1498		- $content_type = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : 'content';
1499		-
1500		- if ( empty($content) ) {
1501		- wp_send_json_error( array( 'message' => 'Content cannot be empty.' ) );
1502		- }
1503		-
1504		- // Get the embedding API key
1505		- $options = get_option('mxchat_options', array());
1506		- $api_key = '';
1507		-
1508		- if ( $bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager') ) {
1509		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
1510		- $api_key = $bot_options['api_key'] ?? '';
1511		- }
1512		- if ( empty($api_key) ) {
1513		- $api_key = $options['api_key'] ?? '';
1514		- }
1515		-
1516		- if ( $data_source === 'pinecone' ) {
1517		- // Pinecone branch. The WP-DB manual-entry delete below must never run here:
1518		- // Pinecone ids are strings, and absint() on a digit-leading md5 hash would
1519		- // yield a real (unrelated) WP row id.
1520		- $raw_vector_id = isset($_POST['entry_id']) ? sanitize_text_field( wp_unslash($_POST['entry_id']) ) : '';
1521		- $is_manual_single = empty($source_url) || strpos($source_url, '_ungrouped_') === 0;
1522		- $is_manual_chunked = strpos($source_url, 'mxchat://') === 0;
1523		-
1524		- if ( $is_manual_single || $is_manual_chunked ) {
1525		- // Manual content: remove the old vectors first, then store as fresh manual
1526		- // content — submit_content_to_db mints a new unique identity (manual_* id
1527		- // for a single vector, an mxchat:// chunk prefix if it now chunks).
1528		- if ( $is_manual_chunked ) {
1529		- // Minted identity: base + chunk vectors share the md5(mxchat://...) prefix.
1530		- MxChat_Utils::delete_chunks_for_url( $source_url, $bot_id );
1531		- } elseif ( ! empty($raw_vector_id) && class_exists('MxChat_Pinecone_Manager') ) {
1532		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
1533		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options( $bot_id );
1534		- if ( ! empty($pinecone_options['mxchat_pinecone_api_key']) && ! empty($pinecone_options['mxchat_pinecone_host']) ) {
1535		- $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
1536		- $raw_vector_id,
1537		- $pinecone_options['mxchat_pinecone_api_key'],
1538		- $pinecone_options['mxchat_pinecone_host'],
1539		- $pinecone_options['mxchat_pinecone_namespace'] ?? ''
1540		- );
1541		- }
1542		- }
1543		- $result = MxChat_Utils::submit_content_to_db( $content, '', $api_key, null, $bot_id, $content_type );
1544		- } else {
1545		- // URL-sourced entry: identity is md5(source_url). submit_content_to_db
1546		- // handles delete-old-chunks → re-chunk → re-embed → store, and sweeps
1547		- // stale chunk vectors when the content now fits in a single vector.
1548		- $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, md5($source_url), $bot_id, $content_type );
1549		- }
1550		- } else {
1551		- global $wpdb;
1552		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
1553		-
1554		- // If source_url is empty but we have an entry_id, look it up
1555		- if ( empty($source_url) && $entry_id > 0 ) {
1556		- $row = $wpdb->get_row( $wpdb->prepare( "SELECT source_url FROM {$table} WHERE id = %d", $entry_id ) );
1557		- if ( $row && ! empty($row->source_url) ) {
1558		- $source_url = $row->source_url;
1559		- }
1560		- }
1561		-
1562		- // For manual entries (no source_url or mxchat:// prefix), delete the old entry by ID first
1563		- // so submit_content_to_db creates a replacement instead of a duplicate
1564		- // Also treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
1565		- $is_legacy_manual = !empty($source_url) && strpos($source_url, 'mxchat.ai') !== false && strpos($source_url, 'mxchat://') !== 0;
1566		- if ( $entry_id > 0 && (empty($source_url) || strpos($source_url, 'mxchat://') === 0 || $is_legacy_manual) ) {
1567		- $wpdb->delete( $table, array( 'id' => $entry_id ), array( '%d' ) );
1568		- // Clear legacy URL so submit_content_to_db generates a unique mxchat:// identifier
1569		- // instead of reusing the shared URL (which would mass-delete other entries with the same URL)
1570		- if ( $is_legacy_manual ) {
1571		- $source_url = '';
1572		- }
1573		- }
1574		-
1575		- // Use the existing submit_content_to_db which handles chunking, Pinecone, and WP DB
1576		- $vector_id = ! empty($source_url) ? md5($source_url) : md5('mxchat_manual_' . $entry_id);
1577		-
1578		- // submit_content_to_db already handles: delete old chunks → re-chunk → re-embed → store
1579		- $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, $vector_id, $bot_id, $content_type );
1580		- }
1581		-
1582		- if ( is_wp_error($result) ) {
1583		- wp_send_json_error( array( 'message' => $result->get_error_message() ) );
1584		- }
1585		-
1586		- wp_send_json_success( array( 'message' => 'Content saved and re-embedded successfully.' ) );
1587		-}
1588		-
1589	775	public function mxchat_get_pdf_processing_status($pdf_url) {
1590	776	$pdf_url = esc_url_raw($pdf_url);
1591	777	$status = get_transient(sanitize_key('mxchat_pdf_status_' . md5($pdf_url)));
1592	778
		@@ -1629,18 +815,24 @@
1629	815	}
1630	816
1631	817
1632	818	public function mxchat_handle_sitemap_submission() {
	819	+ // Start logging the submission process
	820	+ //error_log('[MXCHAT-URL] ===== Starting URL submission process =====');
	821	+
1633	822	// Check if the form was submitted and verify permissions
1634	823	if (!isset($_POST['submit_sitemap']) || !current_user_can('manage_options')) {
	824	+ //error_log('[MXCHAT-URL] Error: Unauthorized access or form not submitted properly');
1635	825	wp_die(esc_html__('Unauthorized access', 'mxchat'));
1636	826	}
1637	827
1638	828	// Verify nonce
	829	+ //error_log('[MXCHAT-URL] Verifying nonce');
1639	830	check_admin_referer('mxchat_submit_sitemap_action', 'mxchat_submit_sitemap_nonce');
1640	831
1641	832	// Validate URL
1642	833	if (!isset($_POST['sitemap_url']) || empty($_POST['sitemap_url'])) {
	834	+ //error_log('[MXCHAT-URL] Error: Empty or missing URL');
1643	835	set_transient('mxchat_admin_notice_error',
1644	836	esc_html__('Please provide a valid URL.', 'mxchat'),
1645	837	30
1646	838	);
		@@ -1648,56 +840,43 @@
1648	840	exit;
1649	841	}
1650	842
1651	843	$submitted_url = esc_url_raw($_POST['sitemap_url']);
	844	+ //error_log('[MXCHAT-URL] Processing URL: ' . $submitted_url);
1652	845
1653		- // Convert Google Drive sharing URLs to direct download URLs
1654		- if ( strpos($submitted_url, 'drive.google.com') !== false ) {
1655		- $file_id = '';
1656		- if ( preg_match('/[?&]id=([a-zA-Z0-9_-]+)/', $submitted_url, $m) ) {
1657		- $file_id = $m[1];
1658		- } elseif ( preg_match('#/file/d/([a-zA-Z0-9_-]+)#', $submitted_url, $m) ) {
1659		- $file_id = $m[1];
1660		- }
1661		- if ( ! empty($file_id) ) {
1662		- $submitted_url = 'https://drive.google.com/uc?export=download&id=' . $file_id;
1663		- }
	846	+ // Validate API key first
	847	+ $options = get_option('mxchat_options');
	848	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	849	+
	850	+ if (strpos($selected_model, 'voyage') === 0) {
	851	+ $api_key = $options['voyage_api_key'] ?? '';
	852	+ $provider_name = 'Voyage AI';
	853	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	854	+ $api_key = $options['gemini_api_key'] ?? '';
	855	+ $provider_name = 'Google Gemini';
	856	+ } else {
	857	+ $api_key = $options['api_key'] ?? '';
	858	+ $provider_name = 'OpenAI';
1664	859	}
1665		-
1666		- // Get bot_id from form submission
1667		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1668		-
1669		- // Get bot-specific options and validate the embedding decision —
1670		- // custom-provider-aware (plan cbd5fd).
1671		- $bot_options = $this->get_bot_options($bot_id);
1672		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
1673		-
1674		- $preflight = MxChat_Utils::embedding_preflight($options);
1675		- if (!$preflight['ok']) {
1676		- set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
	860	+
	861	+ if (empty($api_key)) {
	862	+ $error_message = sprintf(
	863	+ esc_html__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
	864	+ $provider_name
	865	+ );
	866	+ //error_log('[MXCHAT-URL] Error: ' . $error_message);
	867	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	868	+ //error_log('[MXCHAT-URL] Set error transient: ' . $error_message);
1677	869	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1678	870	exit;
1679	871	}
1680		- $api_key = $preflight['api_key'];
1681	872
1682		- // Fetch URL — send an honest, versioned MXChat crawler UA (not a spoofed
1683		- // browser). Stale browser UAs are exactly what WAFs like SiteGround's
1684		- // ModSecurity flag as scrapers, 403-ing the fetch (including PDFs served
1685		- // from the site's own media library, which route through this same call).
1686		- // See mxchat_ingest_user_agent(). Accept is kept for content negotiation;
1687		- // the browser-only Accept-Language fingerprint is dropped so it stays
1688		- // coherent with a bot identity.
1689		- $response = wp_remote_get($submitted_url, array(
1690		- 'timeout' => 30,
1691		- 'sslverify' => false,
1692		- 'user-agent' => mxchat_ingest_user_agent(),
1693		- 'headers' => array(
1694		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1695		- ),
1696		- ));
	873	+ //error_log('[MXCHAT-URL] Fetching URL content');
	874	+ $response = wp_remote_get($submitted_url, array('timeout' => 30));
1697	875
1698	876	if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
1699	877	$error_message = is_wp_error($response) ? $response->get_error_message() : 'HTTP Status: ' . wp_remote_retrieve_response_code($response);
	878	+ //error_log('[MXCHAT-URL] Error fetching URL: ' . $error_message);
1700	879	set_transient('mxchat_admin_notice_error',
1701	880	sprintf(
1702	881	esc_html__('Failed to fetch the URL: %s', 'mxchat'),
1703	882	esc_html($error_message)
		@@ -1708,11 +887,13 @@
1708	887	exit;
1709	888	}
1710	889
1711	890	$content_type = wp_remote_retrieve_header($response, 'content-type');
	891	+ //error_log('[MXCHAT-URL] Content type: ' . $content_type);
1712	892	$body_content = wp_remote_retrieve_body($response);
1713	893
1714	894	if (empty($body_content)) {
	895	+ //error_log('[MXCHAT-URL] Error: Empty response body');
1715	896	set_transient('mxchat_admin_notice_error',
1716	897	esc_html__('Empty response received from URL.', 'mxchat'),
1717	898	30
1718	899	);
		@@ -1718,21 +899,29 @@
1718	899	);
1719	900	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1720	901	exit;
1721	902	}
	903	+ //error_log('[MXCHAT-URL] Retrieved body content length: ' . strlen($body_content) . ' bytes');
1722	904
1723	905	// Handle PDF URL
1724	906	if ($this->mxchat_is_pdf_url($submitted_url, $response)) {
1725		- $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response, $bot_id);
	907	+ //error_log('[MXCHAT-URL] Detected PDF URL, handling PDF for knowledge base');
	908	+ $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response);
	909	+ //error_log('[MXCHAT-URL] PDF handling result: ' . $result);
1726	910
1727		- if ($result === 'queued') {
1728		- set_transient('mxchat_admin_notice_success',
1729		- esc_html__('PDF queued for processing. Processing will start automatically.', 'mxchat'),
	911	+ if ($result === 'scheduled') {
	912	+ set_transient(
	913	+ 'mxchat_last_pdf_url',
	914	+ sanitize_text_field($submitted_url),
	915	+ DAY_IN_SECONDS
	916	+ );
	917	+ set_transient('mxchat_admin_notice_info',
	918	+ esc_html__('PDF processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1730	919	30
1731	920	);
1732	921	} else {
1733	922	set_transient('mxchat_admin_notice_error',
1734		- esc_html__('Failed to queue PDF processing: ', 'mxchat') . esc_html($result),
	923	+ esc_html__('Failed to start PDF processing: ', 'mxchat') . esc_html($result),
1735	924	30
1736	925	);
1737	926	}
1738	927
		@@ -1741,8 +930,9 @@
1741	930	}
1742	931
1743	932	// Handle Sitemap XML
1744	933	if (strpos($content_type, 'xml') !== false || strpos($body_content, '<urlset') !== false) {
	934	+ //error_log('[MXCHAT-URL] Detected XML content, processing as sitemap');
1745	935	libxml_use_internal_errors(true);
1746	936	$xml = simplexml_load_string($body_content);
1747	937	$xml_errors = libxml_get_errors();
1748	938	libxml_clear_errors();
		@@ -1747,8 +937,15 @@
1747	937	$xml_errors = libxml_get_errors();
1748	938	libxml_clear_errors();
1749	939
1750	940	if ($xml === false || !empty($xml_errors)) {
	941	+ //error_log('[MXCHAT-URL] Error: Invalid XML format');
	942	+ if (!empty($xml_errors)) {
	943	+ foreach ($xml_errors as $error) {
	944	+ //error_log('[MXCHAT-URL] XML Error: ' . $error->message);
	945	+ }
	946	+ }
	947	+
1751	948	set_transient('mxchat_admin_notice_error',
1752	949	esc_html__('Invalid sitemap XML. Please provide a valid sitemap.', 'mxchat'),
1753	950	30
1754	951	);
		@@ -1755,30 +952,29 @@
1755	952	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1756	953	exit;
1757	954	}
1758	955
1759		- $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url, $bot_id);
	956	+ //error_log('[MXCHAT-URL] Valid XML found, handling sitemap for knowledge base');
	957	+ $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url);
	958	+ //error_log('[MXCHAT-URL] Sitemap handling result: ' . $result);
1760	959
1761		- if ($result === 'queued') {
1762		- set_transient('mxchat_admin_notice_success',
1763		- esc_html__('Sitemap queued for processing. Processing will start automatically.', 'mxchat'),
	960	+ if ($result === 'scheduled') {
	961	+ set_transient(
	962	+ 'mxchat_last_sitemap_url',
	963	+ sanitize_text_field($submitted_url),
	964	+ DAY_IN_SECONDS
	965	+ );
	966	+ set_transient('mxchat_admin_notice_info',
	967	+ esc_html__('Sitemap processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1764	968	30
1765	969	);
1766	970	} else {
1767		- // Surface the reason the handler already computed (embedding pre-flight,
1768		- // empty sitemap, queue failure). The old message pointed at the status
1769		- // area, which is empty on this path — nothing was ever queued.
1770		- if (is_string($result) && $result !== '') {
1771		- set_transient('mxchat_admin_notice_error',
1772		- esc_html__('Failed to queue sitemap processing: ', 'mxchat') . esc_html($result),
1773		- 30
1774		- );
1775		- } else {
1776		- set_transient('mxchat_admin_notice_error',
1777		- esc_html__('Failed to queue sitemap processing.', 'mxchat'),
1778		- 30
1779		- );
1780		- }
	971	+ // Return to the admin page without a redirect for better error display
	972	+ // The error is already stored in the sitemap status transient
	973	+ set_transient('mxchat_admin_notice_error',
	974	+ esc_html__('Failed to start sitemap processing. Please check the status below for details.', 'mxchat'),
	975	+ 30
	976	+ );
1781	977	}
1782	978
1783	979	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1784	980	exit;
		@@ -1783,50 +979,127 @@
1783	979	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1784	980	exit;
1785	981	}
1786	982
1787		- // Handle Regular URL (single page)
	983	+ // Handle Regular URL
	984	+ //error_log('[MXCHAT-URL] Processing as regular webpage');
1788	985	$page_content = $this->mxchat_extract_main_content($body_content);
	986	+ //error_log('[MXCHAT-URL] Extracted content length: ' . strlen($page_content) . ' bytes');
	987	+
1789	988	$sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	989	+ //error_log('[MXCHAT-URL] Sanitized content length: ' . strlen($sanitized_content) . ' bytes');
1790	990
1791		- //error_log('[MXCHAT-URL-DEBUG] URL: ' . $submitted_url);
1792		- //error_log('[MXCHAT-URL-DEBUG] Extracted page_content length: ' . strlen($page_content));
1793		- //error_log('[MXCHAT-URL-DEBUG] Sanitized content length: ' . strlen($sanitized_content));
1794		- //error_log('[MXCHAT-URL-DEBUG] Content preview: ' . substr($sanitized_content, 0, 500));
1795		-
1796	991	if (empty($sanitized_content)) {
	992	+ //error_log('[MXCHAT-URL] Error: No valid content after sanitization');
	993	+
	994	+ // Set both transients - the error notice and the URL status
1797	995	set_transient('mxchat_admin_notice_error',
1798	996	esc_html__('No valid content found on the provided URL.', 'mxchat'),
1799	997	30
1800	998	);
	999	+
	1000	+ // Set URL status transient
	1001	+ set_transient('mxchat_single_url_status', [
	1002	+ 'url' => $submitted_url,
	1003	+ 'timestamp' => current_time('mysql'),
	1004	+ 'status' => 'failed',
	1005	+ 'error' => esc_html__('No valid content found on the provided URL.', 'mxchat')
	1006	+ ], DAY_IN_SECONDS);
	1007	+
1801	1008	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1802	1009	exit;
1803	1010	}
1804	1011
1805		- // For single URLs, process immediately using submit_content_to_db
1806		- // This handles chunking automatically for large content
1807		- $db_result = MxChat_Utils::submit_content_to_db(
1808		- $sanitized_content,
1809		- $submitted_url,
1810		- $api_key,
1811		- null,
1812		- $bot_id,
1813		- 'url' // content_type
1814		- );
	1012	+ //error_log('[MXCHAT-URL] Generating embedding for content');
	1013	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
1815	1014
1816		- if (is_wp_error($db_result)) {
1817		- $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1015	+ // Check if embedding_vector is a string (error message)
	1016	+ if (is_string($embedding_vector)) {
	1017	+ //error_log('[MXCHAT-URL] Error generating embedding: ' . $embedding_vector);
	1018	+ $error_message = esc_html__('Failed to generate embedding: ', 'mxchat') . esc_html($embedding_vector);
	1019	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1020	+
	1021	+ // Set both transients
1818	1022	set_transient('mxchat_admin_notice_error', $error_message, 30);
1819		- } else {
	1023	+
	1024	+ // Set URL status transient
	1025	+ set_transient('mxchat_single_url_status', [
	1026	+ 'url' => $submitted_url,
	1027	+ 'timestamp' => current_time('mysql'),
	1028	+ 'status' => 'failed',
	1029	+ 'error' => $error_message
	1030	+ ], DAY_IN_SECONDS);
	1031	+
	1032	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1033	+ exit;
	1034	+ }
	1035	+
	1036	+ if (is_array($embedding_vector)) {
	1037	+ //error_log('[MXCHAT-URL] Successfully generated embedding with ' . count($embedding_vector) . ' dimensions');
	1038	+
	1039	+ $db_result = MxChat_Utils::submit_content_to_db(
	1040	+ $sanitized_content,
	1041	+ $submitted_url,
	1042	+ $api_key
	1043	+ );
	1044	+
	1045	+ if (is_wp_error($db_result)) {
	1046	+ //error_log('[MXCHAT-URL] Error: Failed to store content in database: ' . $db_result->get_error_message());
	1047	+ $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1048	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1049	+
	1050	+ // Set both transients
	1051	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1052	+
	1053	+ // Set URL status transient
	1054	+ set_transient('mxchat_single_url_status', [
	1055	+ 'url' => $submitted_url,
	1056	+ 'timestamp' => current_time('mysql'),
	1057	+ 'status' => 'failed',
	1058	+ 'error' => $error_message
	1059	+ ], DAY_IN_SECONDS);
	1060	+
	1061	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1062	+ exit;
	1063	+ }
	1064	+
	1065	+ //error_log('[MXCHAT-URL] Successfully stored content in database');
1820	1066	$success_message = esc_html__('URL content successfully submitted!', 'mxchat');
	1067	+ //error_log('[MXCHAT-URL] Setting success transient: ' . $success_message);
	1068	+
	1069	+ // Set both transients
1821	1070	set_transient('mxchat_admin_notice_success', $success_message, 30);
	1071	+
	1072	+ // Set URL status transient with success
	1073	+ set_transient('mxchat_single_url_status', [
	1074	+ 'url' => $submitted_url,
	1075	+ 'timestamp' => current_time('mysql'),
	1076	+ 'status' => 'complete',
	1077	+ 'content_length' => strlen($sanitized_content),
	1078	+ 'embedding_dimensions' => count($embedding_vector)
	1079	+ ], DAY_IN_SECONDS);
	1080	+
	1081	+ } else {
	1082	+ //error_log('[MXCHAT-URL] Error: Failed to generate embedding. Unexpected result type: ' . gettype($embedding_vector));
	1083	+ $error_message = esc_html__('Failed to generate embedding: Unexpected result type. Please check your API key and try again.', 'mxchat');
	1084	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1085	+
	1086	+ // Set both transients
	1087	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1088	+
	1089	+ // Set URL status transient
	1090	+ set_transient('mxchat_single_url_status', [
	1091	+ 'url' => $submitted_url,
	1092	+ 'timestamp' => current_time('mysql'),
	1093	+ 'status' => 'failed',
	1094	+ 'error' => $error_message
	1095	+ ], DAY_IN_SECONDS);
1822	1096	}
1823	1097
	1098	+ //error_log('[MXCHAT-URL] ===== Completed URL submission process =====');
1824	1099	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1825	1100	exit;
1826	1101	}
1827		-
1828		-
1829	1102	public function mxchat_get_single_url_status() {
1830	1103	$status = get_transient('mxchat_single_url_status');
1831	1104	if (!$status) {
1832	1105	return null;
		@@ -1838,11 +1111,13 @@
1838	1111	}
1839	1112
1840	1113	return $status;
1841	1114	}
1842		-
1843		-public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url, $bot_id = 'default') {
	1115	+public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url) {
	1116	+ // Clear any single URL status when starting sitemap processing
	1117	+ delete_transient('mxchat_single_url_status');
1844	1118	if (!current_user_can('manage_options')) {
	1119	+ //error_log(esc_html__('Unauthorized sitemap processing attempt', 'mxchat'));
1845	1120	return false;
1846	1121	}
1847	1122
1848	1123	try {
		@@ -1851,228 +1126,331 @@
1851	1126	if (!$xml || !is_object($xml)) {
1852	1127	throw new Exception(__('Invalid XML object provided', 'mxchat'));
1853	1128	}
1854	1129
1855		- // Get bot-specific embedding API for validation
1856		- $bot_options = $this->get_bot_options($bot_id);
1857		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	1130	+ // Add embedding validation before processing
	1131	+ // Test embedding with a small sample text to verify API key is working
	1132	+ $test_result = $this->mxchat_generate_embedding("This is a test to verify the embedding API key is working.");
1858	1133
1859		- // Test the embedding API before processing
1860		- $test_phrase = "Test embedding generation for MxChat";
1861		- $test_result = $this->mxchat_generate_embedding($test_phrase, $bot_id);
1862		-
	1134	+ // Check if test_result is a string (error message) rather than an array (valid embedding)
1863	1135	if (is_string($test_result)) {
	1136	+ //error_log('[MXCHAT-URL] Embedding API validation failed: ' . $test_result);
	1137	+
	1138	+ // Store the error in the status transient so it can be displayed later
	1139	+ $status_data = array(
	1140	+ 'total_urls' => 0,
	1141	+ 'processed_urls' => 0,
	1142	+ 'status' => 'error',
	1143	+ 'error' => __('Embedding API validation failed: ', 'mxchat') . $test_result,
	1144	+ 'last_update' => time()
	1145	+ );
	1146	+
	1147	+ set_transient(
	1148	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1149	+ array_map('sanitize_text_field', $status_data),
	1150	+ DAY_IN_SECONDS
	1151	+ );
	1152	+
1864	1153	throw new Exception(__('Embedding API validation failed: ', 'mxchat') . $test_result);
1865	1154	}
1866	1155
	1156	+ // Make sure it's an array (valid embedding)
1867	1157	if (!is_array($test_result)) {
	1158	+ //error_log('[MXCHAT-URL] Embedding API returned unexpected result type: ' . gettype($test_result));
1868	1159	throw new Exception(__('Embedding API returned unexpected result type. Please check your configuration.', 'mxchat'));
1869	1160	}
1870	1161
1871		- // Extract URLs from sitemap
1872		- $urls = array();
	1162	+ $urls = [];
1873	1163	foreach ($xml->url as $url_element) {
1874	1164	$url = esc_url_raw((string)$url_element->loc);
1875	1165	if ($url) {
1876		- $urls[] = array('url' => $url);
	1166	+ $urls[] = $url;
1877	1167	}
1878	1168	}
1879	1169
1880		- $total_urls = count($urls);
	1170	+ $total_urls = absint(count($urls));
1881	1171
1882	1172	if ($total_urls < 1) {
1883	1173	throw new Exception(__('No valid URLs found in sitemap', 'mxchat'));
1884	1174	}
1885	1175
1886		- // Create unique queue ID
1887		- $queue_id = 'sitemap_' . md5($sitemap_url . time());
	1176	+ wp_schedule_single_event(time(), 'mxchat_process_sitemap_urls', array(
	1177	+ 'urls' => $urls,
	1178	+ 'sitemap_url' => $sitemap_url,
	1179	+ 'total_urls' => $total_urls,
	1180	+ 'batch_size' => absint(10),
	1181	+ 'batch_pause' => absint(5)
	1182	+ ));
1888	1183
1889		- // Add URLs to queue
1890		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'url', $urls, $bot_id);
	1184	+ $status_data = array(
	1185	+ 'total_urls' => $total_urls,
	1186	+ 'processed_urls' => 0,
	1187	+ 'status' => 'processing',
	1188	+ 'last_update' => time()
	1189	+ );
1891	1190
1892		- if ($queued_count === 0) {
1893		- throw new Exception(__('Failed to add URLs to processing queue', 'mxchat'));
1894		- }
	1191	+ set_transient(
	1192	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1193	+ array_map('sanitize_text_field', $status_data),
	1194	+ DAY_IN_SECONDS
	1195	+ );
1895	1196
1896		- // Store queue metadata
1897		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $sitemap_url);
1898		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'sitemap');
1899		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_urls);
1900		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
1901		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	1197	+ return __('scheduled', 'mxchat');
1902	1198
1903		- // Store queue ID in transient for status tracking
1904		- set_transient('mxchat_active_queue_sitemap', $queue_id, DAY_IN_SECONDS);
1905		- set_transient('mxchat_last_sitemap_url', $sitemap_url, DAY_IN_SECONDS);
1906		-
1907		- return 'queued';
1908		-
1909		- } catch (Exception $e) {
	1199	+ } catch (\Exception $e) {
1910	1200	$error_message = $e->getMessage();
1911	1201	//error_log(sprintf(esc_html__('Error preparing sitemap for processing: %s', 'mxchat'), esc_html($error_message)));
1912	1202
	1203	+ // Store the sitemap URL and error in transients so they can be displayed
	1204	+ set_transient(
	1205	+ 'mxchat_last_sitemap_url',
	1206	+ sanitize_text_field($sitemap_url),
	1207	+ DAY_IN_SECONDS
	1208	+ );
	1209	+
	1210	+ $status_data = array(
	1211	+ 'total_urls' => 0,
	1212	+ 'processed_urls' => 0,
	1213	+ 'status' => 'error',
	1214	+ 'error' => $error_message,
	1215	+ 'last_update' => time()
	1216	+ );
	1217	+
	1218	+ set_transient(
	1219	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1220	+ array_map('sanitize_text_field', $status_data),
	1221	+ DAY_IN_SECONDS
	1222	+ );
	1223	+
1913	1224	return $error_message;
1914	1225	}
1915		-
1916	1226	}
1917	1227
1918		-/**
1919		- * Remove shortcode tags but preserve the content inside them
1920		- * Example: [vc_column]Hello World[/vc_column] becomes "Hello World"
1921		- *
1922		- * @param string $content The content containing shortcodes
1923		- * @return string Content with shortcode tags removed but inner content preserved
1924		- */
1925		-/**
1926		- * Single-pass HTML entity decode for text entering the knowledge base.
1927		- * The corpus should hold what a human reads: a stored `&amp;` consumes
1928		- * extra tokens, distorts the vector away from the form a visitor's
1929		- * question uses, and can be quoted back verbatim in an answer.
1930		- * Deliberately NOT looped to a fixed point — a stored `&amp;amp;` is a
1931		- * legitimate literal `&amp;` and must not collapse further (data loss).
1932		- * UTF-8 charset keeps multibyte (CJK/RTL) text untouched. Both assembly
1933		- * paths call this at their output points so the treatment cannot drift.
1934		- * (Plan d2c92e.)
1935		- */
1936		-private function mxchat_decode_entities_for_indexing($text) {
1937		- return html_entity_decode((string) $text, ENT_QUOTES | ENT_HTML5, 'UTF-8');
1938		-}
	1228	+public function mxchat_process_sitemap_urls_cron($urls, $sitemap_url, $total_urls, $batch_size, $batch_pause) {
	1229	+ // Validate inputs
	1230	+ $sitemap_url = esc_url_raw($sitemap_url);
	1231	+ $total_urls = absint($total_urls);
	1232	+ $batch_size = absint($batch_size);
	1233	+ $batch_pause = absint($batch_pause);
1939	1234
1940		-/**
1941		- * Price lines for a product's indexed text, pinned to the store's BASE currency.
1942		- *
1943		- * The four product assembly paths each used to call get_woocommerce_currency_symbol()
1944		- * with no argument, which resolves the currency active on the CURRENT request.
1945		- * Multi-currency plugins (CURCY, WOOCS, Aelia, WPML Multicurrency) filter that per
1946		- * request, so whichever currency the store happened to be serving when an import ran
1947		- * was frozen into every product it indexed. The amounts have the mirror problem: the
1948		- * woocommerce_product_get_* filters convert prices in the 'view' context but not in
1949		- * 'edit', so a converted amount could be paired with an unconverted symbol and produce
1950		- * a price that is not merely wrong but incoherent.
1951		- *
1952		- * Base currency option + 'edit' context makes both halves agree and makes the output
1953		- * independent of when the import ran. The currency CODE is emitted alongside the symbol
1954		- * so the model cannot read a bare "$" as USD on a store whose plugin swapped symbols.
1955		- * (Plan 7403ec.)
1956		- */
1957		-private function mxchat_product_price_lines($product) {
1958		- if (!is_object($product) || !method_exists($product, 'get_regular_price')) {
1959		- return '';
	1235	+ if (!is_array($urls) || empty($urls)) {
	1236	+ return;
1960	1237	}
1961	1238
1962		- $currency = get_option('woocommerce_currency');
1963		- $currency = is_string($currency) ? trim($currency) : '';
1964		- $symbol = ($currency !== '')
1965		- ? get_woocommerce_currency_symbol($currency)
1966		- : get_woocommerce_currency_symbol();
1967		- $symbol = $this->mxchat_decode_entities_for_indexing($symbol);
	1239	+ try {
	1240	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	1241	+ $status = get_transient($status_key);
1968	1242
1969		- $regular_price = $product->get_regular_price('edit');
1970		- $sale_price = $product->get_sale_price('edit');
1971		- $price = $product->get_price('edit');
	1243	+ if (!$status || !is_array($status)) {
	1244	+ throw new Exception('Invalid status data retrieved from transient');
	1245	+ }
1972	1246
1973		- $lines = '';
	1247	+ // Initialize failed_urls array if it doesn't exist
	1248	+ if (!isset($status['failed_urls_list']) || !is_array($status['failed_urls_list'])) {
	1249	+ $status['failed_urls_list'] = [];
	1250	+ }
1974	1251
1975		- if (!empty($regular_price)) {
1976		- $lines .= "Price: " . $this->mxchat_format_indexed_price($regular_price, $currency, $symbol) . "\n";
1977		- } elseif (!empty($price)) {
1978		- $lines .= "Price: " . $this->mxchat_format_indexed_price($price, $currency, $symbol) . "\n";
1979		- }
	1252	+ $start_url = absint($status['processed_urls']);
	1253	+ $end_url = min($start_url + $batch_size, $total_urls);
1980	1254
1981		- if (!empty($sale_price) && $sale_price !== $regular_price) {
1982		- $lines .= "Sale Price: " . $this->mxchat_format_indexed_price($sale_price, $currency, $symbol) . "\n";
1983		- }
	1255	+ // Track batch statistics
	1256	+ $batch_stats = [
	1257	+ 'processed' => 0,
	1258	+ 'failed' => 0,
	1259	+ 'last_error' => '',
	1260	+ 'embedding_errors' => 0
	1261	+ ];
1984	1262
1985		- if ($product->is_type('variable')) {
1986		- list($min_price, $max_price) = $this->mxchat_variation_price_range($product);
1987		- if ($min_price !== null && $max_price !== null && (float) $min_price !== (float) $max_price) {
1988		- $lines .= "Price Range: " . $this->mxchat_format_indexed_price($min_price, $currency, $symbol)
1989		- . " - " . $this->mxchat_format_indexed_price($max_price, $currency, $symbol) . "\n";
	1263	+ // Check embedding configuration with first URL (only on first batch)
	1264	+ if ($start_url === 0) {
	1265	+ $test_url = esc_url_raw($urls[0]);
	1266	+ $test_response = wp_remote_get($test_url);
	1267	+
	1268	+ if (!is_wp_error($test_response) && wp_remote_retrieve_response_code($test_response) === 200) {
	1269	+ $test_html = wp_remote_retrieve_body($test_response);
	1270	+ $test_content = $this->mxchat_extract_main_content($test_html);
	1271	+ $test_sanitized = $this->mxchat_sanitize_content_for_api($test_content);
	1272	+
	1273	+ if (!empty($test_sanitized)) {
	1274	+ $test_embedding = $this->mxchat_generate_embedding($test_sanitized);
	1275	+
	1276	+ if (is_string($test_embedding)) {
	1277	+ throw new Exception('Embedding generation failed: ' . $test_embedding);
	1278	+ }
	1279	+
	1280	+ if (!is_array($test_embedding)) {
	1281	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($test_embedding));
	1282	+ }
	1283	+ }
	1284	+ }
1990	1285	}
1991		- }
1992	1286
1993		- return $lines;
1994		-}
	1287	+ for ($i = $start_url; $i < $end_url; $i++) {
	1288	+ $page_url = esc_url_raw($urls[$i]);
	1289	+ $max_retries = 3;
	1290	+ $retry_count = 0;
	1291	+ $url_processed = false;
	1292	+ $last_error = '';
1995	1293
1996		-/**
1997		- * One indexed price amount, labelled with its currency code.
1998		- *
1999		- * "INR 1299.00 (Rs.1299.00)" — the code is what the model should reason from; the symbol
2000		- * is kept so a quoted price still reads naturally. Falls back to the old symbol-only
2001		- * shape when WooCommerce has no base currency configured, and drops the parenthetical
2002		- * when the symbol is absent or IS the code (several currencies have no distinct glyph).
2003		- */
2004		-private function mxchat_format_indexed_price($amount, $currency, $symbol) {
2005		- $amount = (string) $amount;
	1294	+ while (!$url_processed && $retry_count < $max_retries) {
	1295	+ try {
	1296	+ // Attempt to fetch the URL
	1297	+ $page_response = wp_remote_get($page_url, array('timeout' => 30));
2006	1298
2007		- if ($currency === '') {
2008		- return $symbol . $amount;
2009		- }
	1299	+ if (is_wp_error($page_response)) {
	1300	+ throw new Exception('HTTP request failed: ' . $page_response->get_error_message());
	1301	+ }
2010	1302
2011		- if ($symbol === '' || $symbol === $currency) {
2012		- return $currency . ' ' . $amount;
2013		- }
	1303	+ $response_code = wp_remote_retrieve_response_code($page_response);
	1304	+ if ($response_code !== 200) {
	1305	+ throw new Exception('HTTP Status: ' . $response_code);
	1306	+ }
2014	1307
2015		- return $currency . ' ' . $amount . ' (' . $symbol . $amount . ')';
2016		-}
	1308	+ $page_html = wp_remote_retrieve_body($page_response);
	1309	+
	1310	+ if (empty($page_html)) {
	1311	+ throw new Exception('Empty response body');
	1312	+ }
2017	1313
2018		-/**
2019		- * Min/max variation price read from the variations themselves in 'edit' context.
2020		- *
2021		- * get_variation_price() reads WooCommerce's display price cache, which multi-currency
2022		- * plugins populate with converted values — the same defect the rest of this helper
2023		- * exists to remove. Returns raw stored strings (not floats) so the indexed text keeps
2024		- * the store's own price formatting, and (null, null) when no variation carries a price.
2025		- */
2026		-private function mxchat_variation_price_range($product) {
2027		- $min_raw = null;
2028		- $max_raw = null;
2029		- $min_val = null;
2030		- $max_val = null;
	1314	+ $page_content = $this->mxchat_extract_main_content($page_html);
	1315	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
2031	1316
2032		- $children = method_exists($product, 'get_children') ? $product->get_children() : array();
	1317	+ if (empty($sanitized_content)) {
	1318	+ throw new Exception('No valid content found after processing');
	1319	+ }
2033	1320
2034		- foreach ($children as $child_id) {
2035		- $variation = wc_get_product($child_id);
2036		- if (!$variation) {
2037		- continue;
	1321	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	1322	+
	1323	+ if (is_string($embedding_vector)) {
	1324	+ throw new Exception('Embedding generation failed: ' . $embedding_vector);
	1325	+ }
	1326	+
	1327	+ if (!is_array($embedding_vector)) {
	1328	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($embedding_vector));
	1329	+ }
	1330	+
	1331	+ // Submit to database
	1332	+ $options = get_option('mxchat_options');
	1333	+ $submission_result = MxChat_Utils::submit_content_to_db($sanitized_content, $page_url, $options['api_key']);
	1334	+
	1335	+ if (is_wp_error($submission_result)) {
	1336	+ throw new Exception('Database submission failed: ' . $submission_result->get_error_message());
	1337	+ }
	1338	+
	1339	+ // Success!
	1340	+ $url_processed = true;
	1341	+ $batch_stats['processed']++;
	1342	+
	1343	+ } catch (Exception $e) {
	1344	+ $retry_count++;
	1345	+ $last_error = $e->getMessage();
	1346	+
	1347	+ //error_log("URL {$page_url} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	1348	+
	1349	+ // Track embedding errors specifically
	1350	+ if (strpos($last_error, 'Embedding') !== false) {
	1351	+ $batch_stats['embedding_errors']++;
	1352	+ }
	1353	+
	1354	+ if ($retry_count < $max_retries) {
	1355	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	1356	+ sleep(pow(2, $retry_count - 1));
	1357	+ }
	1358	+ }
	1359	+ }
	1360	+
	1361	+ // If URL still not processed after all retries, mark as failed
	1362	+ if (!$url_processed) {
	1363	+ $batch_stats['failed']++;
	1364	+ $batch_stats['last_error'] = $last_error;
	1365	+
	1366	+ // Add to failed URLs list
	1367	+ $status['failed_urls_list'][] = [
	1368	+ 'url' => $page_url,
	1369	+ 'error' => $last_error,
	1370	+ 'time' => time(),
	1371	+ 'retries' => $max_retries
	1372	+ ];
	1373	+
	1374	+ // Limit the number of failed URLs we store to prevent transient size issues
	1375	+ if (count($status['failed_urls_list']) > 100) {
	1376	+ $status['failed_urls_list'] = array_slice($status['failed_urls_list'], -100);
	1377	+ }
	1378	+ }
	1379	+
	1380	+ // Update progress
	1381	+ $status['processed_urls'] = absint($i + 1);
	1382	+ $status['last_update'] = time();
	1383	+ $status['failed_urls'] = absint($status['failed_urls'] ?? 0) + ($url_processed ? 0 : 1);
	1384	+ $status['last_error'] = $batch_stats['last_error'];
	1385	+
	1386	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1387	+
	1388	+ // If we have too many consecutive embedding errors, stop processing
	1389	+ if ($batch_stats['embedding_errors'] >= 10) {
	1390	+ throw new Exception('Too many consecutive embedding failures detected. Please check your API configuration.');
	1391	+ }
2038	1392	}
2039		- $raw = $variation->get_price('edit');
2040		- if ($raw === '' || $raw === null) {
2041		- continue;
	1393	+
	1394	+ // If all URLs in this batch failed, stop processing
	1395	+ if ($batch_stats['processed'] === 0 && $batch_stats['failed'] > 0) {
	1396	+ $status['status'] = 'error';
	1397	+ $status['error'] = sprintf(
	1398	+ 'Processing stopped: %d consecutive failures in batch. Last error: %s',
	1399	+ $batch_stats['failed'],
	1400	+ $batch_stats['last_error']
	1401	+ );
	1402	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1403	+ return;
2042	1404	}
2043		- $val = (float) $raw;
2044		- if ($min_val === null || $val < $min_val) {
2045		- $min_val = $val;
2046		- $min_raw = $raw;
	1405	+
	1406	+ // Update final progress
	1407	+ $status['processed_urls'] = min($end_url, $total_urls);
	1408	+ $status['last_update'] = time();
	1409	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1410	+
	1411	+ // Check if we've processed all URLs
	1412	+ if ($end_url >= $total_urls) {
	1413	+ // All URLs have been processed - mark as complete
	1414	+ $status['status'] = 'complete';
	1415	+ $status['processed_urls'] = $total_urls;
	1416	+
	1417	+ // Add completion summary
	1418	+ $status['completion_summary'] = [
	1419	+ 'total_urls' => $total_urls,
	1420	+ 'successful_urls' => $total_urls - absint($status['failed_urls'] ?? 0),
	1421	+ 'failed_urls' => absint($status['failed_urls'] ?? 0),
	1422	+ 'completion_time' => current_time('mysql')
	1423	+ ];
	1424	+
	1425	+ // Save the completed status (don't delete it - let user dismiss manually)
	1426	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1427	+
	1428	+ // DON'T delete the status transients here - let user dismiss manually
	1429	+ } else {
	1430	+ // Schedule next batch
	1431	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_sitemap_urls', array(
	1432	+ 'urls' => $urls,
	1433	+ 'sitemap_url' => $sitemap_url,
	1434	+ 'total_urls' => $total_urls,
	1435	+ 'batch_size' => $batch_size,
	1436	+ 'batch_pause' => $batch_pause,
	1437	+ ));
2047	1438	}
2048		- if ($max_val === null || $val > $max_val) {
2049		- $max_val = $val;
2050		- $max_raw = $raw;
2051		- }
	1439	+ } catch (\Exception $e) {
	1440	+ $status['status'] = 'error';
	1441	+ $status['error'] = $e->getMessage();
	1442	+ set_transient($status_key, $status, DAY_IN_SECONDS);
2052	1443	}
2053		-
2054		- return array($min_raw, $max_raw);
2055	1444	}
2056		-
2057		-private function strip_shortcode_tags_preserve_content($content) {
2058		- // Single-pass regex removes all shortcode brackets: [tag], [tag attr="val"], [/tag], [tag /]
2059		- // Content between tags is inherently preserved since only brackets are targeted
2060		- $result = preg_replace('/\[\/?\w[\w-]*[^\]]*\]/', '', $content);
2061		- return ($result !== null) ? $result : $content;
2062		-}
2063		-
2064	1445	public function mxchat_sanitize_content_for_api($content) {
2065	1446	//error_log('[MXCHAT-SANITIZE] Original content preview: ' . substr($content, 0, 500) . '...');
2066		-
2067		- // Remove shortcode tags but PRESERVE content inside them
2068		- $content = $this->strip_shortcode_tags_preserve_content($content);
2069		-
	1447	+
2070	1448	// Remove script, style tags, and HTML comments
2071	1449	$content = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $content);
2072	1450	$content = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $content);
2073	1451	$content = preg_replace('/<!--(.|\s)*?-->/', '', $content);
2074		-
	1452	+
2075	1453	// Remove all HTML tags and decode HTML entities
2076	1454	$content = wp_strip_all_tags($content);
2077	1455	$content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5);
2078	1456
		@@ -2098,37 +1476,24 @@
2098	1476
2099	1477	// Ensure valid UTF-8 encoding
2100	1478	$content = wp_check_invalid_utf8($content);
2101	1479
2102		- // Remove extremely long runs with no whitespace (base64 blobs, minified JS).
2103		- // Counts CHARACTERS (/u), and never strips a run containing characters from a
2104		- // script written without spaces — Japanese, Chinese, Thai, Khmer, Lao, Myanmar —
2105		- // where a normal paragraph is legitimately one unbroken run.
2106		- $content = preg_replace_callback('/\S{300,}/u', function ($m) {
2107		- return preg_match('/[\p{Han}\p{Hiragana}\p{Katakana}\p{Thai}\p{Khmer}\p{Lao}\p{Myanmar}]/u', $m[0]) ? $m[0] : ' ';
2108		- }, $content);
2109		-
2110		- // Remove emoji/symbol blocks only — not the whole supplementary plane, which
2111		- // also holds CJK Extension B ideographs used in real Chinese/Japanese names.
2112		- // A ZWJ (U+200D) BETWEEN stripped pictographs is consumed with them, so a
2113		- // family sequence like 👨‍👩‍👧 leaves no invisible zero-width residue behind
2114		- // (the joiner between NON-emoji characters — Hindi conjuncts — is untouched).
2115		- $content = preg_replace('/[\x{1F000}-\x{1F0FF}\x{1F300}-\x{1FAFF}](?:\x{200D}[\x{1F000}-\x{1F0FF}\x{1F300}-\x{1FAFF}])*/u', '', $content);
2116		-
2117		- // There is deliberately NO catch-all character allowlist here (plan 209e57;
2118		- // one existed until 3.2.20). Every genuinely dangerous byte is already gone:
2119		- // control characters, null bytes, invalid UTF-8 and the emoji blocks are all
2120		- // stripped above. The allowlist's only remaining effect was to damage scripts
2121		- // nobody thought to enumerate — Unicode Cf (Format) was missing, so it
2122		- // replaced the zero-width joiner/non-joiner with spaces and silently split
2123		- // Persian words (می‌روم → می روم) and broke Hindi conjuncts (क्‍ष → क् ष).
2124		- // Do not add one back; the failure mode of an allowlist is exactly this.
2125		-
2126		- // Limit to reasonable length if needed (byte limit — MySQL TEXT is byte-sized,
2127		- // but cut on a character boundary so a multibyte char is never split mid-sequence)
	1480	+ // Remove any extremely long strings without spaces (often garbage)
	1481	+ $content = preg_replace('/\S{300,}/', ' ', $content);
	1482	+
	1483	+ // Replace problematic characters that often cause database issues
	1484	+ $content = preg_replace('/[\x{10000}-\x{10FFFF}]/u', '', $content); // Remove emoji and other high Unicode characters
	1485	+
	1486	+ // Replace any remaining potentially problematic characters with spaces
	1487	+ // BUT preserve newlines by temporarily replacing them
	1488	+ $content = str_replace("\n", "NEWLINE_PLACEHOLDER", $content);
	1489	+ $content = preg_replace('/[^\p{L}\p{N}\p{P}\p{Z}\p{Sm}]/u', ' ', $content);
	1490	+ $content = str_replace("NEWLINE_PLACEHOLDER", "\n", $content);
	1491	+
	1492	+ // Limit to reasonable length if needed
2128	1493	$max_length = 65000; // Just under MySQL TEXT field limit
2129	1494	if (strlen($content) > $max_length) {
2130		- $content = mb_strcut($content, 0, $max_length, 'UTF-8');
	1495	+ $content = substr($content, 0, $max_length);
2131	1496	}
2132	1497
2133	1498	//error_log('[MXCHAT-SANITIZE] Sanitized content preview: ' . substr($content, 0, 500) . '...');
2134	1499	return $content;
		@@ -2143,12 +1508,12 @@
2143	1508	@$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'), LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
2144	1509	$xpath = new DOMXPath($dom);
2145	1510
2146	1511	// For debugging purposes
2147		- $debugEnabled = true; // Set to true to enable debugging output
	1512	+ $debugEnabled = false; // Set to true to enable debugging output
2148	1513	$debug = function($message) use ($debugEnabled) {
2149	1514	if ($debugEnabled) {
2150		- //error_log('[MXCHAT-EXTRACT-DEBUG] ' . $message);
	1515	+ //error_log('[MXCHAT-DEBUG] ' . $message);
2151	1516	}
2152	1517	};
2153	1518
2154	1519	// Direct targeting for Gerow theme posts
		@@ -2247,25 +1612,18 @@
2247	1612	'//main',
2248	1613	'//div[contains(@class, "content")]'
2249	1614	];
2250	1615
2251		- // First handle Elementor content - get only leaf widget containers to avoid duplicates
	1616	+ // First handle Elementor content
2252	1617	$debug("Checking for Elementor content");
2253		- // Get widget containers that are direct children of widgets (not nested inside other widget containers)
2254		- $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-widget")]//div[contains(@class, "elementor-widget-container")]');
	1618	+ $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-element")]//div[contains(@class, "elementor-widget-container")]');
2255	1619	if ($elementor_widgets && $elementor_widgets->length > 0) {
2256	1620	$debug("Found Elementor widgets");
2257		- $seen_content = array(); // Track seen content to avoid duplicates
2258	1621	$combined_content = '';
2259	1622	foreach ($elementor_widgets as $widget) {
2260	1623	$widget_content = $dom->saveHTML($widget);
2261	1624	if (!empty($widget_content)) {
2262		- // Create a hash of the content to detect duplicates
2263		- $content_hash = md5($widget_content);
2264		- if (!isset($seen_content[$content_hash])) {
2265		- $seen_content[$content_hash] = true;
2266		- $combined_content .= $widget_content;
2267		- }
	1625	+ $combined_content .= $widget_content;
2268	1626	}
2269	1627	}
2270	1628	if (!empty($combined_content)) {
2271	1629	$debug("Returning Elementor content");
		@@ -2277,14 +1635,15 @@
2277	1635	foreach ($selectors as $selector) {
2278	1636	$debug("Trying selector: " . $selector);
2279	1637	$nodes = $xpath->query($selector);
2280	1638	if ($nodes && $nodes->length > 0) {
2281		- $debug("Found " . $nodes->length . " matches for selector: " . $selector);
2282		- // Only take the FIRST matching node to avoid duplicate content
2283		- // (pages often have nested or multiple containers with same class)
2284		- $content = $dom->saveHTML($nodes->item(0));
	1639	+ $debug("Found matches for selector: " . $selector);
	1640	+ $content = '';
	1641	+ foreach ($nodes as $node) {
	1642	+ $content .= $dom->saveHTML($node);
	1643	+ }
2285	1644	if (!empty($content)) {
2286		- $debug("Returning content from selector: " . $selector . " (first match only)");
	1645	+ $debug("Returning content from selector: " . $selector);
2287	1646	return $content;
2288	1647	}
2289	1648	}
2290	1649	}
		@@ -2308,106 +1667,16 @@
2308	1667	$debug("Returning blog-area section content");
2309	1668	return $content;
2310	1669	}
2311	1670	}
2312		-
2313		- // Generic container selectors for non-CMS sites (like .asp pages)
2314		- $debug("Trying generic container selectors");
2315		- $generic_selectors = [
2316		- '//div[@id="main"]',
2317		- '//div[@id="wrapper"]',
2318		- '//div[@id="page"]',
2319		- '//div[@id="site-content"]',
2320		- '//div[contains(@class, "main-content")]',
2321		- '//div[contains(@class, "page-content")]',
2322		- '//div[contains(@class, "site-content")]',
2323		- ];
2324		-
2325		- foreach ($generic_selectors as $selector) {
2326		- $debug("Trying generic selector: " . $selector);
2327		- $nodes = $xpath->query($selector);
2328		- if ($nodes && $nodes->length > 0) {
2329		- $content = $dom->saveHTML($nodes->item(0));
2330		- if (!empty($content)) {
2331		- $debug("Returning content from generic selector: " . $selector);
2332		- return $content;
2333		- }
2334		- }
2335		- }
2336		-
2337		- // Paragraph-based content detection - find regions with substantial text
2338		- $debug("Trying paragraph-based content detection");
2339		- $paragraphs = $xpath->query('//p[string-length(normalize-space()) > 50]');
2340		- if ($paragraphs && $paragraphs->length >= 3) {
2341		- $debug("Found " . $paragraphs->length . " substantial paragraphs");
2342		- // Collect all substantial paragraphs and their content
2343		- $paragraph_content = '';
2344		- foreach ($paragraphs as $p) {
2345		- $paragraph_content .= $dom->saveHTML($p) . "\n";
2346		- }
2347		- if (!empty($paragraph_content)) {
2348		- $debug("Returning paragraph-based content");
2349		- return $paragraph_content;
2350		- }
2351		- }
2352		-
2353		- // Improved body fallback - strip nav/header/footer elements first
2354		- $debug("Using improved body fallback");
	1671	+
	1672	+ // Fallback: Return the body content if no specific selector matches
	1673	+ $debug("Using body fallback");
2355	1674	$body = $dom->getElementsByTagName('body');
2356	1675	if ($body->length > 0) {
2357		- // Clone the body to avoid modifying the original DOM
2358		- $body_clone = $body->item(0)->cloneNode(true);
2359		-
2360		- // Remove common non-content elements by tag name
2361		- $remove_tags = ['nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript'];
2362		- foreach ($remove_tags as $tag) {
2363		- $elements = $body_clone->getElementsByTagName($tag);
2364		- // Iterate backwards to safely remove elements
2365		- for ($i = $elements->length - 1; $i >= 0; $i--) {
2366		- $el = $elements->item($i);
2367		- if ($el && $el->parentNode) {
2368		- $el->parentNode->removeChild($el);
2369		- }
2370		- }
2371		- }
2372		-
2373		- // Remove elements with common non-content class names using XPath on the cloned body
2374		- $temp_dom = new DOMDocument();
2375		- @$temp_dom->appendChild($temp_dom->importNode($body_clone, true));
2376		- $temp_xpath = new DOMXPath($temp_dom);
2377		-
2378		- $remove_class_patterns = [
2379		- '//*[contains(@class, "nav")]',
2380		- '//*[contains(@class, "menu")]',
2381		- '//*[contains(@class, "sidebar")]',
2382		- '//*[contains(@class, "footer")]',
2383		- '//*[contains(@class, "header")]',
2384		- '//*[contains(@id, "nav")]',
2385		- '//*[contains(@id, "menu")]',
2386		- '//*[contains(@id, "sidebar")]',
2387		- '//*[contains(@id, "footer")]',
2388		- '//*[contains(@id, "header")]',
2389		- ];
2390		-
2391		- foreach ($remove_class_patterns as $pattern) {
2392		- $elements = $temp_xpath->query($pattern);
2393		- if ($elements) {
2394		- for ($i = $elements->length - 1; $i >= 0; $i--) {
2395		- $el = $elements->item($i);
2396		- if ($el && $el->parentNode) {
2397		- $el->parentNode->removeChild($el);
2398		- }
2399		- }
2400		- }
2401		- }
2402		-
2403		- $cleaned_content = $temp_dom->saveHTML();
2404		- if (!empty($cleaned_content)) {
2405		- $debug("Returning cleaned body content");
2406		- return $cleaned_content;
2407		- }
	1676	+ return $dom->saveHTML($body->item(0));
2408	1677	}
2409		-
	1678	+
2410	1679	// Last resort: return the original HTML
2411	1680	$debug("Returning original HTML");
2412	1681	return $html;
2413	1682	} catch (Exception $e) {
		@@ -2459,41 +1728,47 @@
2459	1728	try {
2460	1729	// Verify the request
2461	1730	check_ajax_referer('mxchat_status_nonce', 'nonce');
2462	1731
2463		- // Get active queue IDs
2464		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
2465		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
	1732	+ // Get the status just like in your admin page
	1733	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1734	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
2466	1735
2467		- $sitemap_status = false;
2468		- $pdf_status = false;
	1736	+ $pdf_status = $pdf_url ? $this->mxchat_get_pdf_processing_status($pdf_url) : false;
	1737	+ $sitemap_status = $sitemap_url ? $this->mxchat_get_sitemap_processing_status($sitemap_url) : false;
2469	1738
2470		- // Get sitemap queue status
2471		- if ($sitemap_queue_id) {
2472		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
	1739	+ // Add the PDF URL to the status object
	1740	+ if ($pdf_status && $pdf_url) {
	1741	+ $pdf_status['pdf_url'] = $pdf_url;
2473	1742	}
2474	1743
2475		- // Get PDF queue status
2476		- if ($pdf_queue_id) {
2477		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
2478		- }
	1744	+ // Set the current PDF URL for the manual batch processing button
	1745	+ $current_pdf_url = $pdf_url;
2479	1746
	1747	+ // Check for true processing status, not just presence of status
2480	1748	$is_active_processing =
2481		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
2482		- ($pdf_status && $pdf_status['status'] === 'processing');
	1749	+ ($sitemap_status && isset($sitemap_status['status']) && $sitemap_status['status'] === 'processing') ||
	1750	+ ($pdf_status && isset($pdf_status['status']) && $pdf_status['status'] === 'processing');
2483	1751
	1752	+ // Get single URL status, but only if no processing is active
	1753	+ $single_url_status = !$is_active_processing ? $this->mxchat_get_single_url_status() : false;
	1754	+
	1755	+ // REMOVED: Auto-clearing of completed status - now only done via dismiss button
	1756	+
2484	1757	// Return JSON response with the status data
2485	1758	wp_send_json(array(
2486	1759	'pdf_status' => $pdf_status,
2487	1760	'sitemap_status' => $sitemap_status,
	1761	+ 'single_url_status' => $single_url_status,
2488	1762	'is_processing' => $is_active_processing,
2489		- 'sitemap_queue_id' => $sitemap_queue_id,
2490		- 'pdf_queue_id' => $pdf_queue_id
	1763	+ 'current_pdf_url' => $current_pdf_url
2491	1764	));
2492	1765
2493	1766	} catch (Exception $e) {
	1767	+ // Log the error
2494	1768	//error_log('MxChat Status Update Error: ' . $e->getMessage());
2495	1769
	1770	+ // Return a friendly error response
2496	1771	wp_send_json_error(array(
2497	1772	'message' => 'Error getting status updates: ' . $e->getMessage(),
2498	1773	'status' => 'error'
2499	1774	));
		@@ -2498,1399 +1773,8 @@
2498	1773	'status' => 'error'
2499	1774	));
2500	1775	}
2501	1776	}
2502		-
2503		-/**
2504		- * Helper function to get queue status data
2505		- */
2506		-private function mxchat_get_queue_status_data($queue_id, $type = 'sitemap') {
2507		- global $wpdb;
2508		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
2509		-
2510		- // Get counts by status
2511		- $counts = $wpdb->get_results($wpdb->prepare(
2512		- "SELECT status, COUNT(*) as count
2513		- FROM $table_name
2514		- WHERE queue_id = %s
2515		- GROUP BY status",
2516		- $queue_id
2517		- ), OBJECT_K);
2518		-
2519		- $total = 0;
2520		- $completed = 0;
2521		- $failed = 0;
2522		- $processing = 0;
2523		- $pending = 0;
2524		-
2525		- foreach ($counts as $status => $data) {
2526		- $count = absint($data->count);
2527		- $total += $count;
2528		-
2529		- switch ($status) {
2530		- case 'completed':
2531		- $completed = $count;
2532		- break;
2533		- case 'failed':
2534		- $failed = $count;
2535		- break;
2536		- case 'processing':
2537		- $processing = $count;
2538		- break;
2539		- case 'pending':
2540		- $pending = $count;
2541		- break;
2542		- }
2543		- }
2544		-
2545		- if ($total === 0) {
2546		- return false;
2547		- }
2548		-
2549		- // Calculate percentage
2550		- $percentage = round((($completed + $failed) / $total) * 100);
2551		-
2552		- // Get failed items details (limit to 50)
2553		- $failed_items = array();
2554		- if ($failed > 0) {
2555		- $failed_results = $wpdb->get_results($wpdb->prepare(
2556		- "SELECT item_type, item_data, error_message, attempts, completed_at
2557		- FROM $table_name
2558		- WHERE queue_id = %s
2559		- AND status = 'failed'
2560		- AND attempts >= max_attempts
2561		- ORDER BY id DESC
2562		- LIMIT 50",
2563		- $queue_id
2564		- ));
2565		-
2566		- foreach ($failed_results as $item) {
2567		- $data = json_decode($item->item_data, true);
2568		- $url = $type === 'pdf' ? ($data['pdf_url'] ?? '') : ($data['url'] ?? '');
2569		-
2570		- $failed_items[] = array(
2571		- 'url' => $url,
2572		- 'page' => $type === 'pdf' ? ($data['page_number'] ?? 0) : 0,
2573		- 'error' => $item->error_message,
2574		- 'retries' => $item->attempts,
2575		- 'time' => strtotime($item->completed_at)
2576		- );
2577		- }
2578		- }
2579		-
2580		- // Get queue metadata
2581		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
2582		-
2583		- // Determine if queue is complete
2584		- $is_complete = ($pending === 0 && $processing === 0);
2585		-
2586		- // Get last update time
2587		- $last_update = $wpdb->get_var($wpdb->prepare(
2588		- "SELECT MAX(UNIX_TIMESTAMP(COALESCE(completed_at, started_at, created_at)))
2589		- FROM $table_name
2590		- WHERE queue_id = %s",
2591		- $queue_id
2592		- ));
2593		-
2594		- $last_update_text = $last_update ? human_time_diff($last_update, time()) . ' ' . __('ago', 'mxchat') : __('Just now', 'mxchat');
2595		-
2596		- // Format based on type
2597		- if ($type === 'pdf') {
2598		- return array(
2599		- 'total_pages' => $total,
2600		- 'processed_pages' => $completed + $failed,
2601		- 'failed_pages' => $failed,
2602		- 'percentage' => $percentage,
2603		- 'status' => $is_complete ? 'complete' : 'processing',
2604		- 'last_update' => $last_update_text,
2605		- 'failed_pages_list' => $failed_items,
2606		- 'pdf_url' => $source_url,
2607		- 'queue_id' => $queue_id
2608		- );
2609		- } else {
2610		- return array(
2611		- 'total_urls' => $total,
2612		- 'processed_urls' => $completed + $failed,
2613		- 'failed_urls' => $failed,
2614		- 'percentage' => $percentage,
2615		- 'status' => $is_complete ? 'complete' : 'processing',
2616		- 'last_update' => $last_update_text,
2617		- 'failed_urls_list' => $failed_items,
2618		- 'sitemap_url' => $source_url,
2619		- 'queue_id' => $queue_id
2620		- );
2621		- }
2622		-}
2623		-
2624		-/**
2625		- * Public method to get processing status for both sitemap and PDF queues
2626		- * Used by admin pages to display processing status
2627		- *
2628		- * @return array Array with 'sitemap_status', 'pdf_status', and 'is_processing' keys
2629		- */
2630		-public function mxchat_get_processing_statuses() {
2631		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
2632		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
2633		-
2634		- $sitemap_status = false;
2635		- $pdf_status = false;
2636		-
2637		- if ($sitemap_queue_id) {
2638		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
2639		- }
2640		-
2641		- if ($pdf_queue_id) {
2642		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
2643		- }
2644		-
2645		- $is_processing =
2646		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
2647		- ($pdf_status && $pdf_status['status'] === 'processing');
2648		-
2649		- return array(
2650		- 'sitemap_status' => $sitemap_status,
2651		- 'pdf_status' => $pdf_status,
2652		- 'is_processing' => $is_processing
2653		- );
2654		-}
2655		-
2656		-/**
2657		- * AJAX handler to get recent knowledge entries for real-time table updates
2658		- * UPDATED: Now supports both WordPress DB and Pinecone data sources
2659		- */
2660		-public function ajax_mxchat_get_recent_entries() {
2661		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2662		-
2663		- if (!current_user_can('manage_options')) {
2664		- wp_send_json_error(array('message' => 'Unauthorized'));
2665		- return;
2666		- }
2667		-
2668		- global $wpdb;
2669		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2670		-
2671		- // Get parameters
2672		- $last_id = isset($_POST['last_id']) ? absint($_POST['last_id']) : 0;
2673		- $limit = isset($_POST['limit']) ? min(absint($_POST['limit']), 50) : 10;
2674		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2675		-
2676		- // Check if Pinecone is enabled for this bot
2677		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
2678		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
2679		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2680		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
2681		-
2682		- if ($use_pinecone && $has_pinecone_api) {
2683		- // PINECONE DATA SOURCE - Get grouped entry count (not raw vector count)
2684		- // Use mxchat_fetch_pinecone_records which returns total_unique_entries
2685		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, '', 1, 10, $bot_id, '');
2686		- $total_count = $records['total'] ?? 0;
2687		-
2688		- // For Pinecone, we don't return individual entries during polling
2689		- // (entries are already displayed on page load via mxchat_fetch_pinecone_records)
2690		- // We just return the updated count
2691		- wp_send_json_success(array(
2692		- 'entries' => array(),
2693		- 'total_count' => absint($total_count),
2694		- 'max_id' => $last_id,
2695		- 'data_source' => 'pinecone'
2696		- ));
2697		- return;
2698		- }
2699		-
2700		- // WORDPRESS DB DATA SOURCE
2701		- // Build query to get entries newer than last_id
2702		- $where_clauses = array('1=1');
2703		- $where_values = array();
2704		-
2705		- if ($last_id > 0) {
2706		- $where_clauses[] = 'id > %d';
2707		- $where_values[] = $last_id;
2708		- }
2709		-
2710		- // Note: WordPress DB table doesn't have bot_id column
2711		- // Multi-bot filtering is handled via Pinecone namespaces
2712		-
2713		- $where_sql = implode(' AND ', $where_clauses);
2714		-
2715		- // Get recent entries
2716		- $query = "SELECT id, article_content, source_url, timestamp
2717		- FROM $table_name
2718		- WHERE $where_sql
2719		- ORDER BY id DESC
2720		- LIMIT %d";
2721		-
2722		- $where_values[] = $limit;
2723		-
2724		- $entries = $wpdb->get_results($wpdb->prepare($query, $where_values));
2725		-
2726		- // Get total count of GROUPED entries (by source_url) - matches pagination display
2727		- // Count unique source_urls (excluding mxchat:// internal refs) + count of ungrouped rows
2728		- $total_count = $wpdb->get_var(
2729		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2730		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
2731		- );
2732		-
2733		- // Format entries for response
2734		- $formatted_entries = array();
2735		- $preview_length = 150;
2736		- foreach ($entries as $entry) {
2737		- // Parse chunk metadata using the proper chunker method (same as initial page load)
2738		- if (class_exists('MxChat_Chunker')) {
2739		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($entry->article_content);
2740		- $display_content = $chunk_meta['text'];
2741		- $chunk_metadata = $chunk_meta['metadata'];
2742		- } else {
2743		- $display_content = $entry->article_content;
2744		- $chunk_metadata = array();
2745		- }
2746		-
2747		- $content_preview = mb_strlen($display_content) > $preview_length
2748		- ? mb_substr($display_content, 0, $preview_length) . '...'
2749		- : $display_content;
2750		-
2751		- $formatted_entries[] = array(
2752		- 'id' => $entry->id,
2753		- 'preview' => esc_html($content_preview),
2754		- 'full_content' => wp_kses_post(wpautop($display_content)),
2755		- 'content_length' => mb_strlen($display_content),
2756		- 'preview_length' => $preview_length,
2757		- 'source_url' => $entry->source_url,
2758		- 'has_link' => !empty($entry->source_url) && strpos($entry->source_url, 'mxchat://') !== 0,
2759		- 'chunk_metadata' => $chunk_metadata,
2760		- 'bot_id' => $entry->bot_id ?? 'default',
2761		- 'edit_nonce' => wp_create_nonce('mxchat_edit_entry_nonce'),
2762		- 'delete_nonce' => wp_create_nonce('mxchat_delete_prompt_nonce')
2763		- );
2764		- }
2765		-
2766		- wp_send_json_success(array(
2767		- 'entries' => $formatted_entries,
2768		- 'total_count' => absint($total_count),
2769		- 'max_id' => !empty($entries) ? $entries[0]->id : $last_id,
2770		- 'data_source' => 'wordpress'
2771		- ));
2772		-}
2773		-
2774		-/**
2775		- * Get Pinecone total count from stats API
2776		- * Helper function for ajax_mxchat_get_recent_entries
2777		- */
2778		-private function mxchat_get_pinecone_count_from_stats($pinecone_options) {
2779		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2780		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
2781		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
2782		-
2783		- if (empty($api_key) || empty($host)) {
2784		- return 0;
2785		- }
2786		-
2787		- try {
2788		- $stats_url = "https://{$host}/describe_index_stats";
2789		-
2790		- $response = wp_remote_post($stats_url, array(
2791		- 'headers' => array(
2792		- 'Api-Key' => $api_key,
2793		- 'Content-Type' => 'application/json'
2794		- ),
2795		- 'body' => '{}',
2796		- 'timeout' => 10
2797		- ));
2798		-
2799		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2800		- $body = wp_remote_retrieve_body($response);
2801		- $stats_data = json_decode($body, true);
2802		-
2803		- // If namespace is specified, get count from that specific namespace
2804		- if (!empty($namespace) && isset($stats_data['namespaces'][$namespace]['vectorCount'])) {
2805		- return intval($stats_data['namespaces'][$namespace]['vectorCount']);
2806		- }
2807		-
2808		- // If no namespace specified or namespace not found in response, use total
2809		- return intval($stats_data['totalVectorCount'] ?? 0);
2810		- }
2811		-
2812		- return 0;
2813		-
2814		- } catch (Exception $e) {
2815		- return 0;
2816		- }
2817		-}
2818		-
2819		-/**
2820		- * AJAX handler to refresh Pinecone entries table via AJAX
2821		- * Returns the table HTML for updating the UI without a full page reload
2822		- */
2823		-public function ajax_mxchat_refresh_pinecone_entries() {
2824		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2825		-
2826		- if (!current_user_can('manage_options')) {
2827		- wp_send_json_error(array('message' => 'Unauthorized'));
2828		- return;
2829		- }
2830		-
2831		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2832		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2833		- $per_page = 25;
2834		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2835		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2836		-
2837		- // Get Pinecone manager and options
2838		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
2839		- if (!$pinecone_manager) {
2840		- wp_send_json_error(array('message' => 'Pinecone manager not available'));
2841		- return;
2842		- }
2843		-
2844		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
2845		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2846		- $pinecone_api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2847		-
2848		- if (!$use_pinecone || empty($pinecone_api_key)) {
2849		- wp_send_json_error(array('message' => 'Pinecone not configured'));
2850		- return;
2851		- }
2852		-
2853		- // Fetch records from Pinecone
2854		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, $search_query, $page, $per_page, $bot_id, $content_type_filter);
2855		- $prompts = $records['data'] ?? array();
2856		- $total_records = $records['total'] ?? 0;
2857		-
2858		- // Preprocess Pinecone records — set chunk_metadata and display_content
2859		- // (matches admin-knowledge-page.php preprocessing)
2860		- foreach ($prompts as $prompt) {
2861		- if (isset($prompt->chunk_index) && $prompt->chunk_index !== null) {
2862		- $prompt->chunk_metadata = array(
2863		- 'chunk_index' => intval($prompt->chunk_index),
2864		- 'total_chunks' => isset($prompt->total_chunks) ? intval($prompt->total_chunks) : null,
2865		- 'is_chunked' => isset($prompt->is_chunked) ? (bool) $prompt->is_chunked : true,
2866		- 'source_url' => $prompt->source_url ?? ''
2867		- );
2868		- $prompt->display_content = $prompt->article_content;
2869		- } else {
2870		- $prompt->chunk_metadata = array();
2871		- $prompt->display_content = $prompt->article_content ?? '';
2872		- }
2873		- }
2874		-
2875		- // Group prompts by source_url
2876		- $grouped_prompts = array();
2877		- foreach ($prompts as $prompt) {
2878		- $source_url = '';
2879		- if (!empty($prompt->chunk_metadata['source_url'])) {
2880		- $source_url = $prompt->chunk_metadata['source_url'];
2881		- } elseif (!empty($prompt->source_url)) {
2882		- $source_url = $prompt->source_url;
2883		- }
2884		-
2885		- if (!empty($source_url)) {
2886		- if (!isset($grouped_prompts[$source_url])) {
2887		- $grouped_prompts[$source_url] = array();
2888		- }
2889		- $grouped_prompts[$source_url][] = $prompt;
2890		- } else {
2891		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2892		- }
2893		- }
2894		-
2895		- // Sort each group by chunk_index
2896		- foreach ($grouped_prompts as $source_url => &$group) {
2897		- usort($group, function($a, $b) {
2898		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2899		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2900		- return $index_a - $index_b;
2901		- });
2902		- }
2903		- unset($group);
2904		-
2905		- // Build HTML for the table rows - must match admin-knowledge-page.php structure exactly
2906		- ob_start();
2907		- $display_index = 0;
2908		- $current_page = $page;
2909		- $data_source = 'pinecone';
2910		- $current_bot_id = $bot_id;
2911		- $preview_length = 150;
2912		-
2913		- if (empty($grouped_prompts)) {
2914		- echo '<tr><td colspan="4" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2915		- esc_html_e('No knowledge entries found in Pinecone.', 'mxchat');
2916		- echo '</td></tr>';
2917		- } else {
2918		- foreach ($grouped_prompts as $source_url => $group) {
2919		- $chunk_count = count($group);
2920		- $first_prompt = $group[0];
2921		- $display_index++;
2922		-
2923		- if ($chunk_count > 1) {
2924		- // Multiple chunks - show grouped row with expand button
2925		- $group_id = 'group-' . md5($source_url);
2926		- ?>
2927		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2928		- class="mxchat-chunk-group-header"
2929		- data-source="<?php echo esc_attr($data_source); ?>"
2930		- data-group-id="<?php echo esc_attr($group_id); ?>"
2931		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2932		- <td style="padding: 12px 16px; text-align: center;">
2933		- <input type="checkbox"
2934		- class="mxchat-entry-checkbox"
2935		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2936		- data-source="<?php echo esc_attr($data_source); ?>"
2937		- data-source-url="<?php echo esc_attr($source_url); ?>"
2938		- data-is-group="true"
2939		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2940		- </td>
2941		- <td style="padding: 12px 16px; font-size: 13px;">
2942		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2943		- </td>
2944		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2945		- <div class="mxchat-chunk-group-info">
2946		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2947		- <span class="dashicons dashicons-arrow-right-alt2"></span>
2948		- </button>
2949		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2950		- <span class="mxchat-chunk-preview">
2951		- <?php
2952		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : (isset($first_prompt->article_content) ? $first_prompt->article_content : '');
2953		- $content_preview = mb_substr($parent_content, 0, 100);
2954		- echo esc_html($content_preview . '...');
2955		- ?>
2956		- </span>
2957		- </div>
2958		- </td>
2959		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2960		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2961		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2962		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2963		- <?php esc_html_e('View Source', 'mxchat'); ?>
2964		- </a>
2965		- <?php else : ?>
2966		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2967		- <?php endif; ?>
2968		- </td>
2969		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2970		- <button type="button"
2971		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-inspect-entry-btn"
2972		- data-source-url="<?php echo esc_attr($source_url); ?>"
2973		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2974		- data-data-source="<?php echo esc_attr($data_source); ?>"
2975		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2976		- data-nonce="<?php echo wp_create_nonce('mxchat_inspect_entry_nonce'); ?>"
2977		- title="<?php esc_attr_e('View indexed content', 'mxchat'); ?>">
2978		- <span class="dashicons dashicons-visibility" style="font-size: 14px;"></span>
2979		- </button>
2980		- <button type="button"
2981		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2982		- data-source-url="<?php echo esc_attr($source_url); ?>"
2983		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2984		- data-data-source="<?php echo esc_attr($data_source); ?>"
2985		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2986		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2987		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2988		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2989		- </button>
2990		- <button type="button"
2991		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2992		- data-source-url="<?php echo esc_attr($source_url); ?>"
2993		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2994		- data-data-source="<?php echo esc_attr($data_source); ?>"
2995		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2996		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2997		- style="color: var(--mxch-error);"
2998		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2999		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
3000		- </button>
3001		- </td>
3002		- </tr>
3003		- <?php
3004		- // Render hidden chunk rows
3005		- foreach ($group as $chunk_index => $chunk) {
3006		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
3007		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
3008		- $content = isset($chunk->display_content) ? $chunk->display_content : (isset($chunk->article_content) ? $chunk->article_content : '');
3009		- $content_preview = mb_strlen($content) > $preview_length
3010		- ? mb_substr($content, 0, $preview_length) . '...'
3011		- : $content;
3012		- ?>
3013		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
3014		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
3015		- data-source="<?php echo esc_attr($data_source); ?>"
3016		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
3017		- <td style="padding: 12px 16px; text-align: center;">
3018		- <!-- Checkbox column placeholder for chunks (managed by group) -->
3019		- </td>
3020		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
3021		- <!-- Hidden ID column for chunks -->
3022		- </td>
3023		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3024		- <div class="mxchat-accordion-wrapper">
3025		- <div class="mxchat-content-preview">
3026		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
3027		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
3028		- </span>
3029		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
3030		- <?php if (mb_strlen($content) > $preview_length) : ?>
3031		- <button class="mxchat-expand-toggle" type="button">
3032		- <span class="dashicons dashicons-arrow-down-alt2"></span>
3033		- </button>
3034		- <?php endif; ?>
3035		- </div>
3036		- <div class="mxchat-content-full" style="display: none;">
3037		- <div class="content-view">
3038		- <?php
3039		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
3040		- echo '<div dir="rtl" lang="he" class="rtl-content">';
3041		- echo wp_kses_post(wpautop($content));
3042		- echo '</div>';
3043		- } else {
3044		- echo wp_kses_post(wpautop($content));
3045		- }
3046		- ?>
3047		- </div>
3048		- </div>
3049		- </div>
3050		- </td>
3051		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3052		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
3053		- </td>
3054		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
3055		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
3056		- </td>
3057		- </tr>
3058		- <?php
3059		- }
3060		- } else {
3061		- // Single entry - display normally with accordion
3062		- $prompt = $first_prompt;
3063		- $content = isset($prompt->display_content) ? $prompt->display_content : (isset($prompt->article_content) ? $prompt->article_content : '');
3064		- $content_preview = mb_strlen($content) > $preview_length
3065		- ? mb_substr($content, 0, $preview_length) . '...'
3066		- : $content;
3067		- ?>
3068		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
3069		- data-source="<?php echo esc_attr($data_source); ?>"
3070		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
3071		- <td style="padding: 12px 16px; text-align: center;">
3072		- <input type="checkbox"
3073		- class="mxchat-entry-checkbox"
3074		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
3075		- data-source="<?php echo esc_attr($data_source); ?>"
3076		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
3077		- data-is-group="false"
3078		- data-chunk-count="1">
3079		- </td>
3080		- <td style="padding: 12px 16px; font-size: 13px;">
3081		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
3082		- </td>
3083		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3084		- <div class="mxchat-accordion-wrapper">
3085		- <div class="mxchat-content-preview">
3086		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
3087		- <?php if (mb_strlen($content) > $preview_length) : ?>
3088		- <button class="mxchat-expand-toggle" type="button">
3089		- <span class="dashicons dashicons-arrow-down-alt2"></span>
3090		- </button>
3091		- <?php endif; ?>
3092		- </div>
3093		- <div class="mxchat-content-full" style="display: none;">
3094		- <div class="content-view">
3095		- <?php
3096		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
3097		- echo '<div dir="rtl" lang="he" class="rtl-content">';
3098		- echo wp_kses_post(wpautop($content));
3099		- echo '</div>';
3100		- } else {
3101		- echo wp_kses_post(wpautop($content));
3102		- }
3103		- ?>
3104		- </div>
3105		- </div>
3106		- </div>
3107		- </td>
3108		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3109		- <?php
3110		- $actual_source = $source_url;
3111		- if (strpos($source_url, '_ungrouped_') === 0) {
3112		- $actual_source = $prompt->source_url ?? '';
3113		- }
3114		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
3115		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
3116		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
3117		- <?php esc_html_e('View', 'mxchat'); ?>
3118		- </a>
3119		- <?php else : ?>
3120		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
3121		- <?php endif; ?>
3122		- </td>
3123		- <td style="padding: 12px 16px; white-space: nowrap;">
3124		- <button type="button"
3125		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-inspect-entry-btn"
3126		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
3127		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
3128		- data-data-source="<?php echo esc_attr($data_source); ?>"
3129		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3130		- data-nonce="<?php echo wp_create_nonce('mxchat_inspect_entry_nonce'); ?>"
3131		- title="<?php esc_attr_e('View indexed content', 'mxchat'); ?>">
3132		- <span class="dashicons dashicons-visibility" style="font-size: 14px;"></span>
3133		- </button>
3134		- <button type="button"
3135		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
3136		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
3137		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
3138		- data-data-source="<?php echo esc_attr($data_source); ?>"
3139		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3140		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
3141		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
3142		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
3143		- </button>
3144		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-ajax" data-vector-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_pinecone_prompt_nonce'); ?>" style="color: var(--mxch-error);">
3145		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
3146		- </button>
3147		- </td>
3148		- </tr>
3149		- <?php
3150		- }
3151		- }
3152		- }
3153		- $html = ob_get_clean();
3154		-
3155		- // Generate pagination HTML for Pinecone
3156		- $total_pages = ceil($total_records / $per_page);
3157		- $pagination_html = '';
3158		- if ($total_pages > 1) {
3159		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
3160		-
3161		- // Previous button
3162		- if ($page > 1) {
3163		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
3164		- }
3165		-
3166		- // Page numbers
3167		- $start_page = max(1, $page - 2);
3168		- $end_page = min($total_pages, $page + 2);
3169		-
3170		- if ($start_page > 1) {
3171		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
3172		- if ($start_page > 2) {
3173		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
3174		- }
3175		- }
3176		-
3177		- for ($i = $start_page; $i <= $end_page; $i++) {
3178		- if ($i == $page) {
3179		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
3180		- } else {
3181		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
3182		- }
3183		- }
3184		-
3185		- if ($end_page < $total_pages) {
3186		- if ($end_page < $total_pages - 1) {
3187		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
3188		- }
3189		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
3190		- }
3191		-
3192		- // Next button
3193		- if ($page < $total_pages) {
3194		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
3195		- }
3196		-
3197		- $pagination_html .= '</div>';
3198		- }
3199		-
3200		- wp_send_json_success(array(
3201		- 'html' => $html,
3202		- 'pagination_html' => $pagination_html,
3203		- 'total_count' => $total_records,
3204		- 'total_pages' => $total_pages,
3205		- 'page' => $page,
3206		- 'per_page' => $per_page,
3207		- 'data_source' => 'pinecone'
3208		- ));
3209		-}
3210		-
3211		-/**
3212		- * AJAX handler for pagination - handles both WordPress DB and Pinecone data sources
3213		- * Returns paginated entries without requiring a full page reload
3214		- */
3215		-public function ajax_mxchat_paginate_entries() {
3216		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
3217		-
3218		- if (!current_user_can('manage_options')) {
3219		- wp_send_json_error(array('message' => 'Unauthorized'));
3220		- return;
3221		- }
3222		-
3223		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
3224		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
3225		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
3226		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
3227		- $per_page = 25;
3228		-
3229		- // Check if Pinecone is enabled for this bot
3230		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
3231		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
3232		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3233		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
3234		-
3235		- if ($use_pinecone && $has_pinecone_api) {
3236		- // Delegate to Pinecone pagination handler (pass search params)
3237		- $_POST['page'] = $page;
3238		- $_POST['search'] = $search_query;
3239		- $_POST['content_type'] = $content_type_filter;
3240		- $this->ajax_mxchat_refresh_pinecone_entries();
3241		- return;
3242		- }
3243		-
3244		- // WordPress DB pagination - MUST match initial page load logic exactly
3245		- global $wpdb;
3246		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3247		- $offset = ($page - 1) * $per_page;
3248		-
3249		- // Build WHERE clause for search and content type filtering
3250		- $where_clauses = array();
3251		- $where_values = array();
3252		-
3253		- if ($search_query) {
3254		- $where_clauses[] = "article_content LIKE %s";
3255		- $where_values[] = '%' . $wpdb->esc_like($search_query) . '%';
3256		- }
3257		-
3258		- if ($content_type_filter) {
3259		- switch ($content_type_filter) {
3260		- case 'manual':
3261		- $where_clauses[] = "(source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')";
3262		- break;
3263		- case 'pdf':
3264		- $where_clauses[] = "source_url LIKE '%.pdf'";
3265		- break;
3266		- case 'url':
3267		- $where_clauses[] = "source_url != '' AND source_url IS NOT NULL AND source_url NOT LIKE 'mxchat://%' AND source_url NOT LIKE '%.pdf'";
3268		- break;
3269		- }
3270		- }
3271		-
3272		- $where_sql = !empty($where_clauses) ? 'WHERE ' . implode(' AND ', $where_clauses) : '';
3273		-
3274		- // Count grouped entries with filters applied
3275		- if (!empty($where_values)) {
3276		- $count_args = array_merge($where_values, $where_values);
3277		- $count_query = $wpdb->prepare(
3278		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
3279		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))",
3280		- ...$count_args
3281		- );
3282		- $total_records = $wpdb->get_var($count_query);
3283		- } else if (!empty($where_sql)) {
3284		- // Content type filter only (no search), no prepared values needed
3285		- $total_records = $wpdb->get_var(
3286		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
3287		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))"
3288		- );
3289		- } else {
3290		- // No filters
3291		- $total_records = $wpdb->get_var(
3292		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
3293		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
3294		- );
3295		- }
3296		- $total_pages = ceil($total_records / $per_page);
3297		-
3298		- // Step 1: Get unique source_urls for this page (ordered by latest timestamp) with filters
3299		- if (!empty($where_values)) {
3300		- $query_args = array_merge($where_values, array($per_page, $offset));
3301		- $urls_query = $wpdb->prepare(
3302		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
3303		- {$where_sql}
3304		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
3305		- ...$query_args
3306		- );
3307		- } else if (!empty($where_sql)) {
3308		- $urls_query = $wpdb->prepare(
3309		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
3310		- {$where_sql}
3311		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
3312		- $per_page, $offset
3313		- );
3314		- } else {
3315		- $urls_query = $wpdb->prepare(
3316		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
3317		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
3318		- $per_page, $offset
3319		- );
3320		- }
3321		- $page_urls = $wpdb->get_results($urls_query);
3322		-
3323		- // Step 2: Build list of source_urls to fetch
3324		- $url_list = array();
3325		- $url_order_map = array();
3326		- $order_index = 0;
3327		- foreach ($page_urls as $url_row) {
3328		- $url = $url_row->source_url;
3329		- $url_list[] = $url;
3330		- $url_order_map[$url] = $order_index++;
3331		- }
3332		-
3333		- // Step 3: Fetch all rows for these source_urls (with search filter if applicable)
3334		- $prompts = array();
3335		- if (!empty($url_list)) {
3336		- $placeholders = implode(',', array_fill(0, count($url_list), '%s'));
3337		- if ($search_query) {
3338		- // Include search filter in the final fetch
3339		- $prompts_query = $wpdb->prepare(
3340		- "SELECT id, article_content, source_url, timestamp, role_restriction
3341		- FROM {$table_name}
3342		- WHERE source_url IN ($placeholders) AND article_content LIKE %s
3343		- ORDER BY timestamp DESC",
3344		- ...array_merge($url_list, ['%' . $wpdb->esc_like($search_query) . '%'])
3345		- );
3346		- } else {
3347		- $prompts_query = $wpdb->prepare(
3348		- "SELECT id, article_content, source_url, timestamp, role_restriction
3349		- FROM {$table_name}
3350		- WHERE source_url IN ($placeholders)
3351		- ORDER BY timestamp DESC",
3352		- $url_list
3353		- );
3354		- }
3355		- $prompts = $wpdb->get_results($prompts_query);
3356		- }
3357		-
3358		- // Group prompts by source_url for chunk display
3359		- $grouped_prompts = array();
3360		- foreach ($prompts as $prompt) {
3361		- $source_url = $prompt->source_url ?? '';
3362		-
3363		- // Parse chunk metadata using the proper chunker method (same as initial page load)
3364		- if (class_exists('MxChat_Chunker')) {
3365		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($prompt->article_content);
3366		- $prompt->chunk_metadata = $chunk_meta['metadata'];
3367		- $prompt->display_content = $chunk_meta['text'];
3368		- } else {
3369		- $prompt->chunk_metadata = array();
3370		- $prompt->display_content = $prompt->article_content;
3371		- }
3372		-
3373		- if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0) {
3374		- if (!isset($grouped_prompts[$source_url])) {
3375		- $grouped_prompts[$source_url] = array();
3376		- }
3377		- $grouped_prompts[$source_url][] = $prompt;
3378		- } else {
3379		- // Ungrouped entries
3380		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
3381		- }
3382		- }
3383		-
3384		- // Sort groups by the original URL order (newest first)
3385		- uksort($grouped_prompts, function($a, $b) use ($url_order_map) {
3386		- $order_a = isset($url_order_map[$a]) ? $url_order_map[$a] : PHP_INT_MAX;
3387		- $order_b = isset($url_order_map[$b]) ? $url_order_map[$b] : PHP_INT_MAX;
3388		- return $order_a - $order_b;
3389		- });
3390		-
3391		- // Sort each group internally by chunk_index
3392		- foreach ($grouped_prompts as $source_url => &$group) {
3393		- usort($group, function($a, $b) {
3394		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
3395		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
3396		- return $index_a - $index_b;
3397		- });
3398		- }
3399		- unset($group);
3400		-
3401		- // Build HTML for the table rows
3402		- ob_start();
3403		- $display_index = 0;
3404		- $current_page = $page;
3405		- $data_source = 'wordpress';
3406		- $current_bot_id = $bot_id;
3407		- $preview_length = 150;
3408		-
3409		- if (empty($grouped_prompts)) {
3410		- echo '<tr><td colspan="5" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
3411		- esc_html_e('No knowledge entries found. Use the Import Options to add content.', 'mxchat');
3412		- echo '</td></tr>';
3413		- } else {
3414		- foreach ($grouped_prompts as $source_url => $group) {
3415		- $chunk_count = count($group);
3416		- $first_prompt = $group[0];
3417		- $display_index++;
3418		-
3419		- if ($chunk_count > 1) {
3420		- // Multiple chunks - show grouped row with expand button
3421		- $group_id = 'group-' . md5($source_url);
3422		- ?>
3423		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
3424		- class="mxchat-chunk-group-header"
3425		- data-source="<?php echo esc_attr($data_source); ?>"
3426		- data-group-id="<?php echo esc_attr($group_id); ?>"
3427		- style="border-bottom: 1px solid var(--mxch-card-border);">
3428		- <td style="padding: 12px 16px; text-align: center;">
3429		- <input type="checkbox"
3430		- class="mxchat-entry-checkbox"
3431		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
3432		- data-source="<?php echo esc_attr($data_source); ?>"
3433		- data-source-url="<?php echo esc_attr($source_url); ?>"
3434		- data-is-group="true"
3435		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
3436		- </td>
3437		- <td style="padding: 12px 16px; font-size: 13px;">
3438		- <?php echo esc_html($first_prompt->id); ?>
3439		- </td>
3440		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3441		- <div class="mxchat-chunk-group-info">
3442		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
3443		- <span class="dashicons dashicons-arrow-right-alt2"></span>
3444		- </button>
3445		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
3446		- <span class="mxchat-chunk-preview">
3447		- <?php
3448		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : $first_prompt->article_content;
3449		- $content_preview = mb_substr($parent_content, 0, 100);
3450		- echo esc_html($content_preview . '...');
3451		- ?>
3452		- </span>
3453		- </div>
3454		- </td>
3455		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3456		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
3457		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
3458		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
3459		- <?php esc_html_e('View Source', 'mxchat'); ?>
3460		- </a>
3461		- <?php else : ?>
3462		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
3463		- <?php endif; ?>
3464		- </td>
3465		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
3466		- <?php if ($data_source !== 'pinecone') : ?>
3467		- <button type="button"
3468		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
3469		- data-source-url="<?php echo esc_attr($source_url); ?>"
3470		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
3471		- data-data-source="<?php echo esc_attr($data_source); ?>"
3472		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3473		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
3474		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
3475		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
3476		- </button>
3477		- <?php endif; ?>
3478		- <button type="button"
3479		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
3480		- data-source-url="<?php echo esc_attr($source_url); ?>"
3481		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
3482		- data-data-source="<?php echo esc_attr($data_source); ?>"
3483		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3484		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
3485		- style="color: var(--mxch-error);"
3486		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
3487		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
3488		- </button>
3489		- </td>
3490		- </tr>
3491		- <?php
3492		- // Render hidden chunk rows
3493		- foreach ($group as $chunk_index => $chunk) {
3494		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
3495		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
3496		- $content = isset($chunk->display_content) ? $chunk->display_content : $chunk->article_content;
3497		- $content_preview = mb_strlen($content) > $preview_length
3498		- ? mb_substr($content, 0, $preview_length) . '...'
3499		- : $content;
3500		- ?>
3501		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
3502		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
3503		- data-source="<?php echo esc_attr($data_source); ?>"
3504		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
3505		- <td style="padding: 12px 16px; text-align: center;">
3506		- <!-- Checkbox column placeholder for chunks (managed by group) -->
3507		- </td>
3508		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
3509		- <!-- Hidden ID column for chunks -->
3510		- </td>
3511		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3512		- <div class="mxchat-accordion-wrapper">
3513		- <div class="mxchat-content-preview">
3514		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
3515		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
3516		- </span>
3517		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
3518		- <?php if (mb_strlen($content) > $preview_length) : ?>
3519		- <button class="mxchat-expand-toggle" type="button">
3520		- <span class="dashicons dashicons-arrow-down-alt2"></span>
3521		- </button>
3522		- <?php endif; ?>
3523		- </div>
3524		- <div class="mxchat-content-full" style="display: none;">
3525		- <div class="content-view">
3526		- <?php
3527		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
3528		- echo '<div dir="rtl" lang="he" class="rtl-content">';
3529		- echo wp_kses_post(wpautop($content));
3530		- echo '</div>';
3531		- } else {
3532		- echo wp_kses_post(wpautop($content));
3533		- }
3534		- ?>
3535		- </div>
3536		- </div>
3537		- </div>
3538		- </td>
3539		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3540		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
3541		- </td>
3542		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
3543		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
3544		- </td>
3545		- </tr>
3546		- <?php
3547		- }
3548		- } else {
3549		- // Single entry - display normally with accordion
3550		- $prompt = $first_prompt;
3551		- $content = isset($prompt->display_content) ? $prompt->display_content : $prompt->article_content;
3552		- $content_preview = mb_strlen($content) > $preview_length
3553		- ? mb_substr($content, 0, $preview_length) . '...'
3554		- : $content;
3555		- ?>
3556		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
3557		- data-source="<?php echo esc_attr($data_source); ?>"
3558		- style="border-bottom: 1px solid var(--mxch-card-border);">
3559		- <td style="padding: 12px 16px; text-align: center;">
3560		- <input type="checkbox"
3561		- class="mxchat-entry-checkbox"
3562		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
3563		- data-source="<?php echo esc_attr($data_source); ?>"
3564		- data-source-url="<?php echo esc_attr($source_url); ?>"
3565		- data-is-group="false">
3566		- </td>
3567		- <td style="padding: 12px 16px; font-size: 13px;">
3568		- <?php echo esc_html($prompt->id); ?>
3569		- </td>
3570		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3571		- <div class="mxchat-accordion-wrapper">
3572		- <div class="mxchat-content-preview">
3573		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
3574		- <?php if (mb_strlen($content) > $preview_length) : ?>
3575		- <button class="mxchat-expand-toggle" type="button">
3576		- <span class="dashicons dashicons-arrow-down-alt2"></span>
3577		- </button>
3578		- <?php endif; ?>
3579		- </div>
3580		- <div class="mxchat-content-full" style="display: none;">
3581		- <div class="content-view">
3582		- <?php
3583		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
3584		- echo '<div dir="rtl" lang="he" class="rtl-content">';
3585		- echo wp_kses_post(wpautop($content));
3586		- echo '</div>';
3587		- } else {
3588		- echo wp_kses_post(wpautop($content));
3589		- }
3590		- ?>
3591		- </div>
3592		- </div>
3593		- </div>
3594		- </td>
3595		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3596		- <?php
3597		- $actual_source = $source_url;
3598		- if (strpos($source_url, '_ungrouped_') === 0) {
3599		- $actual_source = $prompt->source_url ?? '';
3600		- }
3601		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
3602		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
3603		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
3604		- <?php esc_html_e('View', 'mxchat'); ?>
3605		- </a>
3606		- <?php else : ?>
3607		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
3608		- <?php endif; ?>
3609		- </td>
3610		- <td style="padding: 12px 16px; white-space: nowrap;">
3611		- <button type="button"
3612		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
3613		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
3614		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
3615		- data-data-source="<?php echo esc_attr($data_source); ?>"
3616		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3617		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
3618		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
3619		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
3620		- </button>
3621		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-wordpress" data-entry-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_wordpress_prompt_nonce'); ?>" style="color: var(--mxch-error);">
3622		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
3623		- </button>
3624		- </td>
3625		- </tr>
3626		- <?php
3627		- }
3628		- }
3629		- }
3630		- $html = ob_get_clean();
3631		-
3632		- // Generate pagination HTML (include search/filter data for subsequent pages)
3633		- $pagination_html = '';
3634		- if ($total_pages > 1) {
3635		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
3636		-
3637		- // Previous button
3638		- if ($page > 1) {
3639		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
3640		- }
3641		-
3642		- // Page numbers
3643		- $start_page = max(1, $page - 2);
3644		- $end_page = min($total_pages, $page + 2);
3645		-
3646		- if ($start_page > 1) {
3647		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
3648		- if ($start_page > 2) {
3649		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
3650		- }
3651		- }
3652		-
3653		- for ($i = $start_page; $i <= $end_page; $i++) {
3654		- if ($i == $page) {
3655		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
3656		- } else {
3657		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
3658		- }
3659		- }
3660		-
3661		- if ($end_page < $total_pages) {
3662		- if ($end_page < $total_pages - 1) {
3663		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
3664		- }
3665		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
3666		- }
3667		-
3668		- // Next button
3669		- if ($page < $total_pages) {
3670		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
3671		- }
3672		-
3673		- $pagination_html .= '</div>';
3674		- }
3675		-
3676		- wp_send_json_success(array(
3677		- 'html' => $html,
3678		- 'pagination_html' => $pagination_html,
3679		- 'total_count' => $total_records,
3680		- 'total_pages' => $total_pages,
3681		- 'page' => $page,
3682		- 'per_page' => $per_page,
3683		- 'data_source' => 'wordpress'
3684		- ));
3685		-}
3686		-
3687		-/**
3688		- * AJAX handler to detect available sitemaps on the site
3689		- * Optimized for speed - only checks primary sitemap indexes first
3690		- */
3691		-public function ajax_mxchat_detect_sitemaps() {
3692		- check_ajax_referer('mxchat_detect_sitemaps_nonce', 'nonce');
3693		-
3694		- if (!current_user_can('manage_options')) {
3695		- wp_send_json_error(array('message' => 'Unauthorized'));
3696		- return;
3697		- }
3698		-
3699		- $site_url = get_site_url();
3700		- $sitemaps = array();
3701		- $found_index = false;
3702		-
3703		- // Only check the main sitemap index files first (much faster)
3704		- // These are the primary entry points that contain sub-sitemaps
3705		- $primary_indexes = array(
3706		- 'sitemap_index.xml' => 'Yoast SEO / Rank Math', // Yoast & Rank Math
3707		- 'wp-sitemap.xml' => 'WordPress Core', // WordPress Core
3708		- 'sitemap.xml' => 'Standard', // Generic/AIOSEO
3709		- );
3710		-
3711		- foreach ($primary_indexes as $path => $source) {
3712		- $url = trailingslashit($site_url) . $path;
3713		-
3714		- $response = wp_remote_head($url, array(
3715		- 'timeout' => 10,
3716		- 'sslverify' => false,
3717		- 'redirection' => 1,
3718		- 'user-agent' => mxchat_ingest_user_agent(),
3719		- ));
3720		-
3721		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3722		- // Found a sitemap index - parse it to get sub-sitemaps
3723		- $sub_sitemaps = $this->parse_sitemap_index($url);
3724		- if (!empty($sub_sitemaps)) {
3725		- $sitemaps[] = array(
3726		- 'url' => $url,
3727		- 'type' => 'index',
3728		- 'source' => $source,
3729		- 'sub_sitemaps' => $sub_sitemaps
3730		- );
3731		- $found_index = true;
3732		- // Found a valid index, no need to check others
3733		- break;
3734		- }
3735		- }
3736		- }
3737		-
3738		- // If no sitemap index found, check for standalone sitemaps
3739		- if (!$found_index) {
3740		- $standalone_sitemaps = array(
3741		- 'post-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
3742		- 'page-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
3743		- );
3744		-
3745		- foreach ($standalone_sitemaps as $path => $info) {
3746		- $url = trailingslashit($site_url) . $path;
3747		-
3748		- $response = wp_remote_head($url, array(
3749		- 'timeout' => 2,
3750		- 'sslverify' => false
3751		- ));
3752		-
3753		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3754		- $sitemaps[] = array(
3755		- 'url' => $url,
3756		- 'type' => $info['type'],
3757		- 'source' => $info['source'],
3758		- 'url_count' => 0 // Skip URL count for speed
3759		- );
3760		- }
3761		- }
3762		- }
3763		-
3764		- wp_send_json_success(array(
3765		- 'sitemaps' => $sitemaps,
3766		- 'site_url' => $site_url
3767		- ));
3768		-}
3769		-
3770		-/**
3771		- * Parse a sitemap index to get sub-sitemaps
3772		- * Optimized: doesn't fetch URL count for each sub-sitemap (too slow)
3773		- */
3774		-private function parse_sitemap_index($url) {
3775		- $sub_sitemaps = array();
3776		-
3777		- $response = wp_remote_get($url, array(
3778		- 'timeout' => 30,
3779		- 'sslverify' => false,
3780		- 'user-agent' => mxchat_ingest_user_agent(),
3781		- 'headers' => array(
3782		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
3783		- ),
3784		- ));
3785		-
3786		- if (is_wp_error($response)) {
3787		- return $sub_sitemaps;
3788		- }
3789		-
3790		- $body = wp_remote_retrieve_body($response);
3791		- if (empty($body)) {
3792		- return $sub_sitemaps;
3793		- }
3794		-
3795		- // Suppress XML errors
3796		- libxml_use_internal_errors(true);
3797		- $xml = simplexml_load_string($body);
3798		- libxml_clear_errors();
3799		-
3800		- if ($xml === false) {
3801		- return $sub_sitemaps;
3802		- }
3803		-
3804		- // Check if it's a sitemap index (contains <sitemap> elements)
3805		- if (isset($xml->sitemap)) {
3806		- foreach ($xml->sitemap as $sitemap) {
3807		- $loc = (string) $sitemap->loc;
3808		- if (!empty($loc)) {
3809		- // Try to determine the type from the URL
3810		- $type = 'content';
3811		- if (strpos($loc, 'category') !== false || strpos($loc, 'tag') !== false || strpos($loc, 'taxonomy') !== false) {
3812		- $type = 'taxonomy';
3813		- } elseif (strpos($loc, 'author') !== false || strpos($loc, 'user') !== false) {
3814		- $type = 'author';
3815		- }
3816		-
3817		- // Skip URL count - too slow to fetch for each sitemap
3818		- $sub_sitemaps[] = array(
3819		- 'url' => $loc,
3820		- 'type' => $type,
3821		- 'url_count' => 0, // Don't fetch - takes too long
3822		- 'name' => basename(parse_url($loc, PHP_URL_PATH))
3823		- );
3824		- }
3825		- }
3826		- }
3827		-
3828		- return $sub_sitemaps;
3829		-}
3830		-
3831		-/**
3832		- * Get URL count from a sitemap
3833		- */
3834		-private function get_sitemap_url_count($url) {
3835		- $response = wp_remote_get($url, array(
3836		- 'timeout' => 30,
3837		- 'sslverify' => false,
3838		- 'user-agent' => mxchat_ingest_user_agent(),
3839		- 'headers' => array(
3840		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
3841		- ),
3842		- ));
3843		-
3844		- if (is_wp_error($response)) {
3845		- return 0;
3846		- }
3847		-
3848		- $body = wp_remote_retrieve_body($response);
3849		- if (empty($body)) {
3850		- return 0;
3851		- }
3852		-
3853		- // Count <url> or <loc> elements
3854		- $count = preg_match_all('/<url>/i', $body, $matches);
3855		- return $count ?: 0;
3856		-}
3857		-
3858		-/**
3859		- * Get sitemaps declared in robots.txt
3860		- */
3861		-private function get_sitemaps_from_robots($site_url) {
3862		- $sitemaps = array();
3863		- $robots_url = trailingslashit($site_url) . 'robots.txt';
3864		-
3865		- $response = wp_remote_get($robots_url, array(
3866		- 'timeout' => 15,
3867		- 'sslverify' => false,
3868		- 'user-agent' => mxchat_ingest_user_agent(),
3869		- ));
3870		-
3871		- if (is_wp_error($response)) {
3872		- return $sitemaps;
3873		- }
3874		-
3875		- $body = wp_remote_retrieve_body($response);
3876		- if (empty($body)) {
3877		- return $sitemaps;
3878		- }
3879		-
3880		- // Find Sitemap: declarations
3881		- if (preg_match_all('/^Sitemap:\s*(.+)$/mi', $body, $matches)) {
3882		- foreach ($matches[1] as $sitemap_url) {
3883		- $sitemap_url = trim($sitemap_url);
3884		- if (filter_var($sitemap_url, FILTER_VALIDATE_URL)) {
3885		- $sitemaps[] = $sitemap_url;
3886		- }
3887		- }
3888		- }
3889		-
3890		- return $sitemaps;
3891		-}
3892		-
3893	1777	public function mxchat_stop_processing() {
3894	1778	// Verify permissions
3895	1779	if (!current_user_can('manage_options')) {
3896	1780	wp_die(esc_html__('Unauthorized access', 'mxchat'));
		@@ -3898,52 +1782,28 @@
3898	1782
3899	1783	// Verify nonce
3900	1784	check_admin_referer('mxchat_stop_processing_action', 'mxchat_stop_processing_nonce');
3901	1785
3902		- global $wpdb;
3903		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
3904		-
3905		- // Get active queue IDs
3906		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
3907		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
3908		-
3909		- // Delete all pending items from active queues
3910		- if ($sitemap_queue_id) {
3911		- $wpdb->delete(
3912		- $table_name,
3913		- array(
3914		- 'queue_id' => $sitemap_queue_id,
3915		- 'status' => 'pending'
3916		- ),
3917		- array('%s', '%s')
3918		- );
3919		-
3920		- delete_transient('mxchat_active_queue_sitemap');
	1786	+ // Get the last sitemap URL and clear its transient
	1787	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
	1788	+ if ($sitemap_url) {
	1789	+ delete_transient('mxchat_sitemap_status_' . md5($sitemap_url));
3921	1790	delete_transient('mxchat_last_sitemap_url');
3922	1791	}
3923		-
3924		- if ($pdf_queue_id) {
3925		- // Get PDF path before deleting
3926		- $pdf_path = $this->mxchat_get_queue_meta($pdf_queue_id, 'pdf_path');
3927		-
3928		- $wpdb->delete(
3929		- $table_name,
3930		- array(
3931		- 'queue_id' => $pdf_queue_id,
3932		- 'status' => 'pending'
3933		- ),
3934		- array('%s', '%s')
3935		- );
3936		-
3937		- // Delete PDF file
3938		- if ($pdf_path && file_exists($pdf_path)) {
3939		- wp_delete_file($pdf_path);
3940		- }
3941		-
3942		- delete_transient('mxchat_active_queue_pdf');
	1792	+
	1793	+ // Get the last PDF URL and clear its transient
	1794	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1795	+ if ($pdf_url) {
	1796	+ delete_transient('mxchat_pdf_status_' . md5($pdf_url));
3943	1797	delete_transient('mxchat_last_pdf_url');
3944	1798	}
3945	1799
	1800	+ // Unschedule any pending sitemap events
	1801	+ $timestamp = wp_next_scheduled('mxchat_process_sitemap_urls');
	1802	+ if ($timestamp) {
	1803	+ wp_unschedule_event($timestamp, 'mxchat_process_sitemap_urls');
	1804	+ }
	1805	+
3946	1806	// Redirect back with a success message
3947	1807	set_transient('mxchat_admin_notice_success',
3948	1808	esc_html__('Processing has been stopped successfully.', 'mxchat'),
3949	1809	30
		@@ -3950,12 +1810,8 @@
3950	1810	);
3951	1811	wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
3952	1812	exit;
3953	1813	}
3954		-
3955		-/**
3956		- * Get content list for processing
3957		- */
3958	1814	public function ajax_mxchat_get_content_list() {
3959	1815	// Verify the nonce
3960	1816	check_ajax_referer('mxchat_content_selector_nonce', 'nonce');
3961	1817
		@@ -3963,9 +1819,9 @@
3963	1819	wp_send_json_error(__('Unauthorized access', 'mxchat'));
3964	1820	}
3965	1821
3966	1822	$page = isset($_GET['page']) ? absint($_GET['page']) : 1;
3967		- $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 100;
	1823	+ $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 20;
3968	1824	$search = isset($_GET['search']) ? sanitize_text_field($_GET['search']) : '';
3969	1825	$post_type = isset($_GET['post_type']) ? sanitize_text_field($_GET['post_type']) : 'all';
3970	1826	$post_status = isset($_GET['post_status']) ? sanitize_text_field($_GET['post_status']) : 'publish';
3971	1827	$processed_filter = isset($_GET['processed_filter']) ? sanitize_text_field($_GET['processed_filter']) : 'all';
		@@ -3978,47 +1834,20 @@
3978	1834	'orderby' => 'date',
3979	1835	'order' => 'DESC',
3980	1836	);
3981	1837
3982		- // Handle post types - IMPROVED VERSION
	1838	+ // Handle post types
3983	1839	if ($post_type !== 'all') {
3984	1840	$args['post_type'] = $post_type;
3985	1841	} else {
3986		- // Get all available post types that might contain content
3987		- $all_post_types = array();
	1842	+ // Default to post and page if we can't get post types
	1843	+ $args['post_type'] = array('post', 'page');
3988	1844
3989		- // First get all public post types
3990		- $public_types = get_post_types(array('public' => true), 'names');
3991		- $all_post_types = array_merge($all_post_types, $public_types);
3992		-
3993		- // Add common forum/community post types
3994		- $forum_types = array('topic', 'reply', 'forum', 'wpforo_topic', 'wpforo_post');
3995		- foreach ($forum_types as $forum_type) {
3996		- if (post_type_exists($forum_type)) {
3997		- $all_post_types[] = $forum_type;
3998		- }
	1845	+ // Try to get public post types
	1846	+ $public_types = $this->mxchat_get_public_post_types();
	1847	+ if (is_array($public_types) && !empty($public_types)) {
	1848	+ $args['post_type'] = array_keys($public_types);
3999	1849	}
4000		-
4001		- // Add other commonly used post types
4002		- $common_types = array('product', 'job_listing', 'event', 'portfolio');
4003		- foreach ($common_types as $common_type) {
4004		- if (post_type_exists($common_type)) {
4005		- $all_post_types[] = $common_type;
4006		- }
4007		- }
4008		-
4009		- // Remove duplicates and ensure we have at least some post types
4010		- $all_post_types = array_unique($all_post_types);
4011		-
4012		- if (empty($all_post_types)) {
4013		- // Fallback to basic post types
4014		- $all_post_types = array('post', 'page');
4015		- }
4016		-
4017		- $args['post_type'] = $all_post_types;
4018		-
4019		- // Debug logging to see what post types are being queried
4020		- //error_log('MxChat Debug: Querying post types: ' . implode(', ', $all_post_types));
4021	1850	}
4022	1851
4023	1852	if (!empty($search)) {
4024	1853	$args['s'] = $search;
		@@ -4023,58 +1852,37 @@
4023	1852	if (!empty($search)) {
4024	1853	$args['s'] = $search;
4025	1854	}
4026	1855
4027		- // Get processed data from storage
	1856	+ // ================================
	1857	+ // FIXED: Check only the ACTIVE storage method
	1858	+ // ================================
	1859	+
4028	1860	$processed_data = array();
4029	1861
	1862	+ $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
4030	1863	$pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	1864	+ $pinecone_manager->mxchat_refresh_after_new_content($pinecone_options);
4031	1865	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
4032	1866
4033	1867	if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
4034		- // Get fresh data from Pinecone - no caching
	1868	+ // ONLY check Pinecone if it's enabled
4035	1869	$processed_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
4036	1870	} else {
4037		- // WordPress DB checking with better URL matching for all post types
	1871	+ // ONLY check WordPress DB if Pinecone is not enabled
4038	1872	global $wpdb;
4039	1873	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
4040		- $processed_items = $wpdb->get_results("SELECT id, source_url, article_content, timestamp FROM {$table_name}");
4041		-
4042		- // Group items by source_url to count chunks
4043		- $url_chunk_counts = array();
4044		- $url_latest_timestamp = array();
4045		- $url_first_id = array();
4046		-
	1874	+ $processed_items = $wpdb->get_results("SELECT id, source_url, timestamp FROM {$table_name}");
	1875	+
4047	1876	if (!empty($processed_items)) {
4048	1877	foreach ($processed_items as $item) {
4049		- $url = $item->source_url;
4050		- if (empty($url)) continue;
4051		-
4052		- // Count chunks per URL
4053		- if (!isset($url_chunk_counts[$url])) {
4054		- $url_chunk_counts[$url] = 0;
4055		- $url_latest_timestamp[$url] = $item->timestamp;
4056		- $url_first_id[$url] = $item->id;
4057		- }
4058		- $url_chunk_counts[$url]++;
4059		-
4060		- // Track latest timestamp
4061		- if (strtotime($item->timestamp) > strtotime($url_latest_timestamp[$url])) {
4062		- $url_latest_timestamp[$url] = $item->timestamp;
4063		- }
4064		- }
4065		-
4066		- // Now build processed_data with chunk counts
4067		- foreach ($url_chunk_counts as $url => $chunk_count) {
4068		- $post_id = $this->mxchat_url_to_post_id_improved($url);
4069		-
	1878	+ $post_id = url_to_postid($item->source_url);
4070	1879	if ($post_id) {
4071	1880	$processed_data[$post_id] = array(
4072		- 'db_id' => $url_first_id[$url],
4073		- 'timestamp' => $url_latest_timestamp[$url],
4074		- 'url' => $url,
4075		- 'source' => 'wordpress',
4076		- 'chunk_count' => $chunk_count
	1881	+ 'db_id' => $item->id,
	1882	+ 'timestamp' => $item->timestamp,
	1883	+ 'url' => $item->source_url,
	1884	+ 'source' => 'wordpress'
4077	1885	);
4078	1886	}
4079	1887	}
4080	1888	}
		@@ -4079,8 +1887,10 @@
4079	1887	}
4080	1888	}
4081	1889	}
4082	1890
	1891	+ // ================================
	1892	+
4083	1893	// Get processed IDs as a simple array for in_array checks
4084	1894	$processed_ids = array_keys($processed_data);
4085	1895
4086	1896	// Handle processed/unprocessed filter
		@@ -4122,14 +1932,8 @@
4122	1932	$db_record_id = $item_data['db_id'];
4123	1933	}
4124	1934	}
4125	1935
4126		- // Get chunk count for this item
4127		- $chunk_count = 0;
4128		- if ($is_processed && isset($processed_data[$id]['chunk_count'])) {
4129		- $chunk_count = intval($processed_data[$id]['chunk_count']);
4130		- }
4131		-
4132	1936	$content_items[] = array(
4133	1937	'id' => $id,
4134	1938	'title' => get_the_title(),
4135	1939	'permalink' => get_permalink(),
		@@ -4140,10 +1944,9 @@
4140	1944	'word_count' => $word_count,
4141	1945	'already_processed' => $is_processed,
4142	1946	'processed_date' => $processed_date,
4143	1947	'db_record_id' => $db_record_id,
4144		- 'data_source' => $data_source,
4145		- 'chunk_count' => $chunk_count
	1948	+ 'data_source' => $data_source
4146	1949	);
4147	1950	}
4148	1951	wp_reset_postdata();
4149	1952	}
		@@ -4159,178 +1962,8 @@
4159	1962	wp_send_json_success($response);
4160	1963	exit;
4161	1964	}
4162	1965
4163		-
4164		-/**
4165		- * This function handles various WooCommerce URL formats and permalink structures
4166		- */
4167		-private function mxchat_url_to_post_id_improved($url) {
4168		- // First try the standard WordPress function
4169		- $post_id = url_to_postid($url);
4170		-
4171		- if ($post_id > 0) {
4172		- return $post_id;
4173		- }
4174		-
4175		- // If that fails, try more aggressive URL matching
4176		- // Remove trailing slashes and query parameters for better matching
4177		- $clean_url = rtrim($url, '/');
4178		- $clean_url = strtok($clean_url, '?'); // Remove query parameters
4179		-
4180		- // Try again with cleaned URL
4181		- $post_id = url_to_postid($clean_url);
4182		- if ($post_id > 0) {
4183		- return $post_id;
4184		- }
4185		-
4186		- // For bbPress forum topics, try extracting slug from URL
4187		- if (strpos($url, '/topic/') !== false || strpos($url, '/forums/') !== false) {
4188		- // Handle bbPress URLs: /forums/topic/topic-name/
4189		- if (preg_match('/\/forums\/topic\/([^\/\?]+)/', $url, $matches)) {
4190		- $topic_slug = $matches[1];
4191		-
4192		- // Look up topic by slug
4193		- $topic = get_page_by_path($topic_slug, OBJECT, 'topic');
4194		- if ($topic) {
4195		- return $topic->ID;
4196		- }
4197		-
4198		- // Alternative method: query by post_name
4199		- global $wpdb;
4200		- $post_id = $wpdb->get_var($wpdb->prepare(
4201		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
4202		- $topic_slug
4203		- ));
4204		-
4205		- if ($post_id) {
4206		- return intval($post_id);
4207		- }
4208		- }
4209		-
4210		- // Handle simpler topic URLs: /topic/topic-name/
4211		- if (preg_match('/\/topic\/([^\/\?]+)/', $url, $matches)) {
4212		- $topic_slug = $matches[1];
4213		-
4214		- global $wpdb;
4215		- $post_id = $wpdb->get_var($wpdb->prepare(
4216		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
4217		- $topic_slug
4218		- ));
4219		-
4220		- if ($post_id) {
4221		- return intval($post_id);
4222		- }
4223		- }
4224		- }
4225		-
4226		- // For WooCommerce products
4227		- if (strpos($url, '/product/') !== false || strpos($url, 'product=') !== false) {
4228		- // Extract product slug from various URL formats
4229		- $product_slug = '';
4230		-
4231		- // Handle pretty permalinks: /product/product-name/
4232		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
4233		- $product_slug = $matches[1];
4234		- }
4235		- // Handle query parameters: ?product=product-name
4236		- elseif (preg_match('/[\?&]product=([^&]+)/', $url, $matches)) {
4237		- $product_slug = $matches[1];
4238		- }
4239		-
4240		- if (!empty($product_slug)) {
4241		- // Look up product by slug
4242		- $product = get_page_by_path($product_slug, OBJECT, 'product');
4243		- if ($product) {
4244		- return $product->ID;
4245		- }
4246		-
4247		- // Alternative method: query by post_name
4248		- global $wpdb;
4249		- $post_id = $wpdb->get_var($wpdb->prepare(
4250		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'product' AND post_status = 'publish'",
4251		- $product_slug
4252		- ));
4253		-
4254		- if ($post_id) {
4255		- return intval($post_id);
4256		- }
4257		- }
4258		- }
4259		-
4260		- // Generic approach: try to extract slug and match against all post types
4261		- $parsed_url = wp_parse_url($clean_url);
4262		- $path = $parsed_url['path'] ?? '';
4263		-
4264		- if (!empty($path)) {
4265		- // Get the last part of the path as potential slug
4266		- $path_parts = array_filter(explode('/', trim($path, '/')));
4267		- $potential_slug = end($path_parts);
4268		-
4269		- if (!empty($potential_slug)) {
4270		- global $wpdb;
4271		-
4272		- // Try to find any post with this slug
4273		- $post_id = $wpdb->get_var($wpdb->prepare(
4274		- "SELECT ID FROM {$wpdb->posts}
4275		- WHERE post_name = %s
4276		- AND post_status IN ('publish', 'closed', 'private')
4277		- AND post_type NOT IN ('revision', 'attachment', 'nav_menu_item')
4278		- ORDER BY CASE
4279		- WHEN post_type = 'post' THEN 1
4280		- WHEN post_type = 'page' THEN 2
4281		- WHEN post_type = 'topic' THEN 3
4282		- WHEN post_type = 'product' THEN 4
4283		- ELSE 5
4284		- END
4285		- LIMIT 1",
4286		- $potential_slug
4287		- ));
4288		-
4289		- if ($post_id) {
4290		- return intval($post_id);
4291		- }
4292		- }
4293		- }
4294		-
4295		- // ADDITIONAL: Try direct database lookup by URL variations
4296		- global $wpdb;
4297		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
4298		-
4299		- // Try variations of the URL (with/without trailing slash, http/https)
4300		- $url_variations = array(
4301		- $url,
4302		- rtrim($url, '/'),
4303		- $url . '/',
4304		- str_replace('http://', 'https://', $url),
4305		- str_replace('https://', 'http://', $url),
4306		- str_replace('http://', 'https://', rtrim($url, '/')),
4307		- str_replace('https://', 'http://', rtrim($url, '/'))
4308		- );
4309		-
4310		- // Remove duplicates
4311		- $url_variations = array_unique($url_variations);
4312		-
4313		- foreach ($url_variations as $variation) {
4314		- $existing_record = $wpdb->get_row($wpdb->prepare(
4315		- "SELECT id, source_url FROM $table_name WHERE source_url = %s",
4316		- $variation
4317		- ));
4318		-
4319		- if ($existing_record) {
4320		- // Try to get post ID from this stored URL
4321		- $stored_post_id = url_to_postid($existing_record->source_url);
4322		- if ($stored_post_id > 0) {
4323		- return $stored_post_id;
4324		- }
4325		- }
4326		- }
4327		-
4328		- return 0; // No match found
4329		-}
4330		-/**
4331		- * Process selected content via AJAX
4332		- */
4333	1966	public function ajax_mxchat_process_selected_content() {
4334	1967	// Basic request validation
4335	1968	if (!check_ajax_referer('mxchat_content_selector_nonce', 'nonce', false)) {
4336	1969	wp_send_json_error('Invalid nonce');
		@@ -4354,16 +1987,8 @@
4354	1987	wp_send_json_error('No content selected');
4355	1988	exit;
4356	1989	}
4357	1990
4358		- // Get bot_id from request
4359		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
4360		-
4361		- // ACF→PDF extraction is an install-level setting (Knowledge → ACF Fields,
4362		- // plan 11720c). The import modal shows a passive status line pointing
4363		- // there; the old per-batch checkbox and its remembered default are gone.
4364		- $extract_acf_pdfs = get_option('mxchat_acf_pdf_extraction', '0') === '1';
4365		-
4366	1991	// Process only ONE post at a time to avoid request size issues
4367	1992	$post_id = reset($post_ids);
4368	1993	$post = get_post($post_id);
4369	1994
		@@ -4370,73 +1995,54 @@
4370	1995	if (!$post) {
4371	1996	wp_send_json_error('Post not found');
4372	1997	exit;
4373	1998	}
4374		-
4375		- /**
4376		- * Allow developers to modify post data before processing into the knowledge base.
4377		- * Applied on BOTH content-preparation paths (this manual bulk import and the
4378		- * auto-sync path in mxchat_handle_post_update) with the same signature, so a
4379		- * callback registered once covers every indexing route. Purely additive —
4380		- * zero behaviour change when unhooked.
4381		- *
4382		- * @param WP_Post $post The post about to be indexed.
4383		- * @param string $bot_id Bot context for this import.
4384		- */
4385		- $post = apply_filters('mxchat_before_process_post', $post, $bot_id);
4386		- if (!($post instanceof WP_Post)) {
4387		- $post = get_post($post_id); // defend against a bad callback return
	1999	+
	2000	+ // Get minimal content
	2001	+ $content = $post->post_title . "\n\n" . wp_strip_all_tags($post->post_content);
	2002	+ $content = substr($content, 0, 10000); // Limit content size
	2003	+
	2004	+ // Get API key with proper model detection
	2005	+ $options = get_option('mxchat_options');
	2006	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	2007	+
	2008	+ if (strpos($selected_model, 'voyage') === 0) {
	2009	+ $api_key = $options['voyage_api_key'] ?? '';
	2010	+ $provider_name = 'Voyage AI';
	2011	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	2012	+ $api_key = $options['gemini_api_key'] ?? '';
	2013	+ $provider_name = 'Google Gemini';
	2014	+ } else {
	2015	+ $api_key = $options['api_key'] ?? '';
	2016	+ $provider_name = 'OpenAI';
4388	2017	}
4389		-
4390		- // Assemble the indexable text via the shared post-kind assembler (a3d60c).
4391		- // Bulk import reads raw post fields, has always included product custom tabs,
4392		- // and passes the install-level ACF→PDF option.
4393		- $prepared = $this->mxchat_prepare_post_content_for_indexing($post_id, $post, array(
4394		- 'read_display' => false,
4395		- 'extract_acf_pdfs' => $extract_acf_pdfs,
4396		- 'include_product_tabs' => true,
4397		- ));
4398		- $content = $prepared['content'];
4399		- $pdf_extracted_count = $prepared['pdf_extracted_count'];
4400		-
4401		- // Debug logging for WordPress Import content
4402		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Post ID: ' . $post_id . ' Title: ' . $post->post_title);
4403		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Raw post_content length: ' . strlen($post->post_content));
4404		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Final content length: ' . strlen($content));
4405		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Content preview: ' . substr($content, 0, 300));
4406		-
4407		- // Note: Removed 10,000 char limit - chunking now handles large content properly
4408		-
4409		- // Get bot-specific embedding decision — custom-provider-aware (plan cbd5fd)
4410		- $bot_options = $this->get_bot_options($bot_id);
4411		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
4412		-
4413		- $preflight = MxChat_Utils::embedding_preflight($options);
4414		- if (!$preflight['ok']) {
4415		- MxChat_Admin::mxchat_log_debug('api_error', $preflight['reason'] . ' (knowledge processing)');
4416		- wp_send_json_error($preflight['reason']);
	2018	+
	2019	+ if (empty($api_key)) {
	2020	+ wp_send_json_error($provider_name . ' API key not configured');
4417	2021	exit;
4418	2022	}
4419		- $api_key = $preflight['api_key'];
4420	2023
4421	2024	$source_url = get_permalink($post_id);
4422	2025	$vector_id = md5($source_url); // Vector ID for Pinecone
4423	2026
4424		- // Check for existing content in bot-specific storage
	2027	+ // ================================
	2028	+ // FIXED: Check for existing content in ONLY the active storage method
	2029	+ // ================================
	2030	+
4425	2031	$is_update = false;
4426	2032
4427		- // Get bot-specific Pinecone configuration
4428		- $bot_pinecone_config = $this->get_bot_pinecone_config($bot_id);
4429		- $use_pinecone = !empty($bot_pinecone_config) && ($bot_pinecone_config['use_pinecone'] ?? false);
	2033	+ // Check if Pinecone is enabled
	2034	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	2035	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
4430	2036
4431		- if ($use_pinecone && !empty($bot_pinecone_config['api_key'])) {
4432		- // Check Pinecone for this bot
4433		- $pinecone_data = $this->mxchat_get_pinecone_processed_content($bot_pinecone_config);
	2037	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2038	+ // ONLY check Pinecone if it's enabled
	2039	+ $pinecone_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
4434	2040	if (isset($pinecone_data[$post_id])) {
4435	2041	$is_update = true;
4436	2042	}
4437	2043	} else {
4438		- // Check WordPress DB (same as before since it's shared)
	2044	+ // ONLY check WordPress DB if Pinecone is not enabled
4439	2045	global $wpdb;
4440	2046	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
4441	2047	$existing_record = $wpdb->get_row($wpdb->prepare(
4442	2048	"SELECT id FROM $table_name WHERE source_url = %s",
		@@ -4446,53 +2052,53 @@
4446	2052	if ($existing_record) {
4447	2053	$is_update = true;
4448	2054	}
4449	2055	}
4450		-
4451		- // UPDATED 2.5.6: Determine content type based on post_type
4452		- $post_type = $post->post_type;
4453		- $content_type = 'content'; // Default fallback
4454		-
4455		- // Map WordPress post types to content types
4456		- switch ($post_type) {
4457		- case 'post':
4458		- $content_type = 'post';
4459		- break;
4460		- case 'page':
4461		- $content_type = 'page';
4462		- break;
4463		- case 'product':
4464		- $content_type = 'product';
4465		- break;
4466		- default:
4467		- // For custom post types, use the post type name
4468		- $content_type = sanitize_key($post_type);
4469		- break;
4470		- }
4471		-
4472		- // Use the centralized utility function with bot_id and content_type
	2056	+
	2057	+ // Use the centralized utility function for storage
4473	2058	$result = MxChat_Utils::submit_content_to_db(
4474		- $content,
4475		- $source_url,
	2059	+ $content,
	2060	+ $source_url,
4476	2061	$api_key,
4477		- $vector_id,
4478		- $bot_id,
4479		- $content_type
	2062	+ $vector_id
4480	2063	);
4481		-
	2064	+
4482	2065	if (is_wp_error($result)) {
4483		- MxChat_Admin::mxchat_log_debug('storage_error', 'Knowledge storage failed: ' . $result->get_error_message(), array('source_url' => $source_url));
4484	2066	wp_send_json_error('Storage failed: ' . $result->get_error_message());
4485	2067	exit;
4486	2068	}
4487		-
4488		- // Automatically apply role restriction based on tags
4489		- $this->apply_role_restriction_to_post($post_id, $source_url);
4490		-
	2069	+
	2070	+ // ================================
	2071	+ // UPDATE: Only update caches if Pinecone is enabled
	2072	+ // ================================
	2073	+
	2074	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2075	+ // Update vector ID cache for improved fetching
	2076	+ $this->mxchat_update_pinecone_vector_cache($vector_id);
	2077	+
	2078	+ // Update local processed content cache for immediate UI feedback
	2079	+ $pinecone_cache = get_option('mxchat_pinecone_processed_cache', array());
	2080	+ $pinecone_cache[$post_id] = array(
	2081	+ 'db_id' => $vector_id,
	2082	+ 'processed_date' => 'Just now',
	2083	+ 'url' => $source_url,
	2084	+ 'source' => 'pinecone',
	2085	+ 'timestamp' => current_time('timestamp')
	2086	+ );
	2087	+ update_option('mxchat_pinecone_processed_cache', $pinecone_cache);
	2088	+
	2089	+ // Also update the general processed content cache
	2090	+ $processed_cache = get_option('mxchat_processed_content_cache', array());
	2091	+ $processed_cache[$post_id] = array(
	2092	+ 'db_id' => $vector_id,
	2093	+ 'timestamp' => current_time('timestamp'),
	2094	+ 'url' => $source_url,
	2095	+ 'source' => 'pinecone'
	2096	+ );
	2097	+ update_option('mxchat_processed_content_cache', $processed_cache);
	2098	+ }
	2099	+
4491	2100	$operation_type = $is_update ? 'update' : 'new';
4492		-
4493		- // Count ACF fields for debugging
4494		- $acf_field_count = $prepared['acf_fields_found'];
4495	2101
4496	2102	// Success response with minimal data
4497	2103	wp_send_json_success(array(
4498	2104	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
		@@ -4498,102 +2104,27 @@
4498	2104	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
4499	2105	'post_id' => $post_id,
4500	2106	'title' => $post->post_title,
4501	2107	'operation_type' => $operation_type,
4502		- 'vector_id' => $vector_id,
4503		- 'acf_fields_found' => $acf_field_count,
4504		- 'pdf_extracted_count' => (int) $pdf_extracted_count,
4505		- 'content_preview' => substr($content, 0, 100) . '...',
4506		- 'bot_id' => $bot_id
	2108	+ 'vector_id' => $vector_id, // Include vector ID for debugging
	2109	+ 'cache_updated' => $use_pinecone // Indicate if cache was updated
4507	2110	));
4508	2111	exit;
4509	2112	}
4510	2113
4511		-private function apply_role_restriction_to_post($post_id, $source_url) {
4512		- // Get tag-role mappings
4513		- $mappings = get_option('mxchat_tag_role_mappings', array());
4514		-
4515		- if (empty($mappings)) {
4516		- return; // No mappings, leave as public
4517		- }
4518		-
4519		- // Get all tags for the post
4520		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
4521		-
4522		- if (empty($post_tags)) {
4523		- return; // No tags, leave as public
4524		- }
4525		-
4526		- // Determine the highest role restriction based on tags
4527		- $highest_role = 'public';
4528		- $role_hierarchy = array(
4529		- 'public' => 0,
4530		- 'logged_in' => 1,
4531		- 'subscriber' => 2,
4532		- 'contributor' => 3,
4533		- 'author' => 4,
4534		- 'editor' => 5,
4535		- 'administrator' => 6
4536		- );
4537		-
4538		- foreach ($post_tags as $tag_slug) {
4539		- if (isset($mappings[$tag_slug])) {
4540		- $role = $mappings[$tag_slug];
4541		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
4542		- $highest_role = $role;
4543		- }
4544		- }
4545		- }
4546		-
4547		- // If no restricted tags found, return (leave as public)
4548		- if ($highest_role === 'public') {
4549		- return;
4550		- }
4551		-
4552		- // Update the role restriction in the database
4553		- global $wpdb;
4554		-
4555		- // Check if using Pinecone
4556		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
4557		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
4558		-
4559		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
4560		- // Update Pinecone role restriction
4561		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
4562		- $vector_id = md5($source_url);
4563		-
4564		- $wpdb->replace(
4565		- $roles_table,
4566		- array(
4567		- 'vector_id' => $vector_id,
4568		- 'role_restriction' => $highest_role,
4569		- 'updated_at' => current_time('mysql')
4570		- ),
4571		- array('%s', '%s', '%s')
4572		- );
4573		- } else {
4574		- // Update WordPress DB
4575		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
4576	2114
4577		- $wpdb->update(
4578		- $table_name,
4579		- array('role_restriction' => $highest_role),
4580		- array('source_url' => $source_url),
4581		- array('%s'),
4582		- array('%s')
4583		- );
4584		- }
4585	2115
4586		- // The entry's restriction just changed — keep the OpenAI Vector Store
4587		- // mirror consistent: non-public pulls the file (file_search has no
4588		- // per-role filtering), public re-mirrors it (plan 15b5c6).
4589		- if (class_exists('MxChat_Vectorstore_Manager')) {
4590		- MxChat_Vectorstore_Manager::handle_role_change($source_url, 'default', $highest_role);
4591		- }
4592		-}
4593		-
	2116	+ /**
	2117	+ * Updates cache with new vector ID if absent
	2118	+ */
	2119	+ public function mxchat_update_pinecone_vector_cache($vector_id) {
	2120	+ $cached_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2121	+ if (!in_array($vector_id, $cached_ids)) {
	2122	+ $cached_ids[] = $vector_id;
	2123	+ update_option('mxchat_pinecone_vector_ids_cache', $cached_ids);
	2124	+ }
	2125	+ }
4594	2126	public function mxchat_get_public_post_types() {
4595		- // Get all public post types
4596	2127	$post_types = get_post_types(array('public' => true), 'objects');
4597	2128	$post_type_options = array();
4598	2129
4599	2130	foreach ($post_types as $post_type) {
		@@ -4599,45 +2130,51 @@
4599	2130	foreach ($post_types as $post_type) {
4600	2131	$post_type_options[$post_type->name] = $post_type->label;
4601	2132	}
4602	2133
4603		- // Also include common forum/community post types that might not be marked as public
4604		- $additional_types = array(
4605		- 'topic' => 'Forum Topics (bbPress)',
4606		- 'reply' => 'Forum Replies (bbPress)',
4607		- 'forum' => 'Forums (bbPress)',
4608		- 'wpforo_topic' => 'wpForo Topics',
4609		- 'wpforo_post' => 'wpForo Posts'
4610		- );
4611		-
4612		- foreach ($additional_types as $type_name => $type_label) {
4613		- if (post_type_exists($type_name) && !isset($post_type_options[$type_name])) {
4614		- $post_type_options[$type_name] = $type_label;
4615		- }
4616		- }
4617		-
4618	2134	return $post_type_options;
4619	2135	}
	2136	+public function mxchat_get_pinecone_processed_content($pinecone_options) {
	2137	+ //error_log('=== DEBUG: Starting mxchat_get_pinecone_processed_content ===');
	2138	+
	2139	+ // First check local cache for immediate updates
	2140	+ $cached_data = get_option('mxchat_pinecone_processed_cache', array());
	2141	+ //error_log('DEBUG: Found ' . count($cached_data) . ' items in local cache');
4620	2142
4621		-/**
4622		- * Retrieves processed content from Pinecone API
4623		- */
4624		-public function mxchat_get_pinecone_processed_content($pinecone_options) {
4625	2143	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4626	2144	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
	2145	+
	2146	+ //error_log('DEBUG: API key present: ' . (!empty($api_key) ? 'YES' : 'NO'));
	2147	+ //error_log('DEBUG: Host: ' . $host);
4627	2148
4628	2149	if (empty($api_key) || empty($host)) {
4629		- return array();
	2150	+ //error_log('DEBUG: Missing API credentials, returning cached data only');
	2151	+ return $cached_data;
4630	2152	}
4631	2153
4632	2154	$pinecone_data = array();
4633	2155
4634	2156	try {
4635		- // Always get fresh data from Pinecone
4636		- $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
	2157	+ // Method 1: Try to get vectors using cached vector IDs first
	2158	+ $cached_vector_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2159	+ //error_log('DEBUG: Found ' . count($cached_vector_ids) . ' cached vector IDs');
4637	2160
4638		- // Method 2: Final fallback - try stats endpoint (if available)
	2161	+ if (!empty($cached_vector_ids)) {
	2162	+ //error_log('DEBUG: Trying to fetch by cached vector IDs...');
	2163	+ $pinecone_data = $this->mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $cached_vector_ids);
	2164	+ //error_log('DEBUG: Fetch by IDs returned ' . count($pinecone_data) . ' items');
	2165	+ }
	2166	+
	2167	+ // Method 2: If no cached IDs or fetch failed, use scanning approach
4639	2168	if (empty($pinecone_data)) {
	2169	+ //error_log('DEBUG: Trying scanning approach...');
	2170	+ $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
	2171	+ //error_log('DEBUG: Scanning returned ' . count($pinecone_data) . ' items');
	2172	+ }
	2173	+
	2174	+ // Method 3: Final fallback - try stats endpoint
	2175	+ if (empty($pinecone_data)) {
	2176	+ //error_log('DEBUG: Trying stats endpoint...');
4640	2177	$stats_url = "https://{$host}/describe_index_stats";
4641	2178
4642	2179	$response = wp_remote_post($stats_url, array(
4643	2180	'headers' => array(
		@@ -4650,73 +2187,101 @@
4650	2187
4651	2188	if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
4652	2189	$body = wp_remote_retrieve_body($response);
4653	2190	$stats_data = json_decode($body, true);
	2191	+ //error_log('DEBUG: Pinecone stats: ' . print_r($stats_data, true));
	2192	+ } else {
	2193	+ if (is_wp_error($response)) {
	2194	+ //error_log('DEBUG: Stats endpoint error: ' . $response->get_error_message());
	2195	+ } else {
	2196	+ //error_log('DEBUG: Stats endpoint failed with code: ' . wp_remote_retrieve_response_code($response));
	2197	+ }
4654	2198	}
4655	2199	}
4656	2200
4657	2201	} catch (Exception $e) {
4658		- // Log error but return fresh data only
	2202	+ //error_log('DEBUG: Exception in get_pinecone_processed_content: ' . $e->getMessage());
4659	2203	}
4660	2204
4661		- return $pinecone_data;
	2205	+ // Merge cached data with Pinecone data
	2206	+ $merged_data = $pinecone_data;
	2207	+
	2208	+ foreach ($cached_data as $post_id => $cache_item) {
	2209	+ $cache_timestamp = $cache_item['timestamp'] ?? 0;
	2210	+ $time_diff = current_time('timestamp') - $cache_timestamp;
	2211	+
	2212	+ if ($time_diff < 300) { // 5 minutes = 300 seconds
	2213	+ $merged_data[$post_id] = $cache_item;
	2214	+ } else {
	2215	+ if (!isset($merged_data[$post_id])) {
	2216	+ $merged_data[$post_id] = $cache_item;
	2217	+ }
	2218	+ }
	2219	+ }
	2220	+
	2221	+ //error_log('DEBUG: Final merged data count: ' . count($merged_data));
	2222	+ //error_log('=== DEBUG: End mxchat_get_pinecone_processed_content ===');
	2223	+
	2224	+ return $merged_data;
4662	2225	}
	2226	+
4663	2227	public function mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $vector_ids) {
	2228	+ //error_log('=== DEBUG: Starting mxchat_fetch_pinecone_vectors_by_ids ===');
	2229	+
4664	2230	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4665	2231	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
4666		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
4667	2232
4668	2233	if (empty($api_key) || empty($host) || empty($vector_ids)) {
	2234	+ //error_log('DEBUG: Missing parameters for fetch by IDs');
4669	2235	return array();
4670	2236	}
4671	2237
4672	2238	try {
4673		- // NOTE (plan 793b82): /vectors/fetch is a GET endpoint with the ids
4674		- // repeated in the query string (ids=a&ids=b — http_build_query would
4675		- // emit ids[0]=a); the old POST here was answered 200-with-an-empty-body,
4676		- // which read as "nothing indexed". Chunked at 100 ids to stay well
4677		- // under the measured HTTP 414 URL-length boundary.
4678		- $vectors = array();
4679		- foreach (array_chunk(array_values($vector_ids), 100) as $chunk) {
4680		- $fetch_query = array();
4681		- foreach ($chunk as $fetch_vid) {
4682		- $fetch_query[] = 'ids=' . rawurlencode($fetch_vid);
4683		- }
4684		- if (!empty($namespace)) {
4685		- $fetch_query[] = 'namespace=' . rawurlencode($namespace);
4686		- }
	2239	+ $fetch_url = "https://{$host}/vectors/fetch";
	2240	+ //error_log('DEBUG: Fetch URL: ' . $fetch_url);
	2241	+ //error_log('DEBUG: Fetching ' . count($vector_ids) . ' vector IDs');
4687	2242
4688		- $response = wp_remote_get("https://{$host}/vectors/fetch?" . implode('&', $fetch_query), array(
4689		- 'headers' => array(
4690		- 'Api-Key' => $api_key,
4691		- 'accept' => 'application/json'
4692		- ),
4693		- 'timeout' => 30
4694		- ));
	2243	+ // Pinecone fetch API allows fetching specific vectors by ID
	2244	+ $fetch_data = array(
	2245	+ 'ids' => array_values($vector_ids)
	2246	+ );
4695	2247
4696		- if (is_wp_error($response)) {
4697		- error_log('MxChat Pinecone: mxchat_fetch_pinecone_vectors_by_ids GET failed: ' . $response->get_error_message());
4698		- continue;
4699		- }
	2248	+ $response = wp_remote_post($fetch_url, array(
	2249	+ 'headers' => array(
	2250	+ 'Api-Key' => $api_key,
	2251	+ 'Content-Type' => 'application/json'
	2252	+ ),
	2253	+ 'body' => json_encode($fetch_data),
	2254	+ 'timeout' => 30
	2255	+ ));
4700	2256
4701		- if (wp_remote_retrieve_response_code($response) !== 200) {
4702		- error_log('MxChat Pinecone: mxchat_fetch_pinecone_vectors_by_ids GET returned HTTP ' . wp_remote_retrieve_response_code($response));
4703		- continue;
4704		- }
	2257	+ if (is_wp_error($response)) {
	2258	+ //error_log('DEBUG: Fetch by IDs WP error: ' . $response->get_error_message());
	2259	+ return array();
	2260	+ }
4705	2261
4706		- $data = json_decode(wp_remote_retrieve_body($response), true);
4707		- if (isset($data['vectors']) && is_array($data['vectors'])) {
4708		- $vectors += $data['vectors'];
4709		- }
	2262	+ $response_code = wp_remote_retrieve_response_code($response);
	2263	+ //error_log('DEBUG: Fetch response code: ' . $response_code);
	2264	+
	2265	+ if ($response_code !== 200) {
	2266	+ $error_body = wp_remote_retrieve_body($response);
	2267	+ //error_log('DEBUG: Fetch failed with body: ' . $error_body);
	2268	+ return array();
4710	2269	}
4711	2270
4712		- if (empty($vectors)) {
	2271	+ $body = wp_remote_retrieve_body($response);
	2272	+ $data = json_decode($body, true);
	2273	+
	2274	+ //error_log('DEBUG: Fetch response structure: ' . print_r(array_keys($data), true));
	2275	+
	2276	+ if (!isset($data['vectors'])) {
	2277	+ //error_log('DEBUG: No vectors key in response');
4713	2278	return array();
4714	2279	}
4715	2280
4716	2281	$processed_data = array();
4717	2282
4718		- foreach ($vectors as $vector_id => $vector_data) {
	2283	+ foreach ($data['vectors'] as $vector_id => $vector_data) {
4719	2284	$metadata = $vector_data['metadata'] ?? array();
4720	2285	$source_url = $metadata['source_url'] ?? '';
4721	2286
4722	2287	if (!empty($source_url)) {
		@@ -4751,49 +2316,16 @@
4751	2316	return array();
4752	2317	}
4753	2318	}
4754	2319
4755		-/**
4756		- * Get embedding dimensions based on the selected model.
4757		- */
4758		-private function mxchat_get_embedding_dimensions() {
4759		- $options = get_option('mxchat_options', array());
4760		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4761		-
4762		- $model_dimensions = array(
4763		- 'text-embedding-ada-002' => 1536,
4764		- 'text-embedding-3-small' => 1536,
4765		- 'text-embedding-3-large' => 3072,
4766		- 'voyage-2' => 1024,
4767		- 'voyage-large-2' => 1536,
4768		- 'voyage-3-large' => 2048,
4769		- 'gemini-embedding-001' => 1536,
4770		- );
4771		-
4772		- if (strpos($selected_model, 'voyage-3-large') === 0) {
4773		- $custom_dimensions = $options['voyage_output_dimension'] ?? 2048;
4774		- return intval($custom_dimensions);
4775		- }
4776		-
4777		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4778		- $custom_dimensions = $options['gemini_output_dimension'] ?? 1536;
4779		- return intval($custom_dimensions);
4780		- }
4781		-
4782		- return $model_dimensions[$selected_model] ?? 1536;
4783		-}
4784		-
4785		-/**
4786		- * Scan Pinecone for processed content
4787		- */
4788	2320	public function mxchat_scan_pinecone_for_processed_content($pinecone_options) {
	2321	+ //error_log('=== DEBUG: Starting mxchat_scan_pinecone_for_processed_content ===');
	2322	+
4789	2323	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4790	2324	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
4791		- // plan 793b82: this scan was namespace-blind — on a namespaced setup it
4792		- // surveyed the default namespace and reported the wrong content as indexed.
4793		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
4794	2325
4795	2326	if (empty($api_key) || empty($host)) {
	2327	+ //error_log('DEBUG: Missing API credentials for scanning');
4796	2328	return array();
4797	2329	}
4798	2330
4799	2331	try {
		@@ -4800,18 +2332,17 @@
4800	2332	// Use multiple random vectors to get better coverage
4801	2333	$all_matches = array();
4802	2334	$seen_ids = array();
4803	2335
4804		- // Get correct dimensions for the configured embedding model
4805		- $dimensions = $this->mxchat_get_embedding_dimensions();
4806		-
4807	2336	// Try 3 different random vectors to get better coverage
4808	2337	for ($i = 0; $i < 3; $i++) {
	2338	+ //error_log('DEBUG: Scanning attempt ' . ($i + 1) . '/3');
	2339	+
4809	2340	$query_url = "https://{$host}/query";
4810	2341
4811	2342	// Generate a random unit vector instead of zeros
4812	2343	$random_vector = array();
4813		- for ($j = 0; $j < $dimensions; $j++) {
	2344	+ for ($j = 0; $j < 1536; $j++) {
4814	2345	$random_vector[] = (rand(-1000, 1000) / 1000.0);
4815	2346	}
4816	2347
4817	2348	// Normalize the vector to unit length
		@@ -4826,12 +2357,8 @@
4826	2357	'topK' => 10000,
4827	2358	'vector' => $random_vector
4828	2359	);
4829	2360
4830		- if (!empty($namespace)) {
4831		- $query_data['namespace'] = $namespace;
4832		- }
4833		-
4834	2361	$response = wp_remote_post($query_url, array(
4835	2362	'headers' => array(
4836	2363	'Api-Key' => $api_key,
4837	2364	'Content-Type' => 'application/json'
		@@ -4840,14 +2367,18 @@
4840	2367	'timeout' => 30
4841	2368	));
4842	2369
4843	2370	if (is_wp_error($response)) {
	2371	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' WP error: ' . $response->get_error_message());
4844	2372	continue;
4845	2373	}
4846	2374
4847	2375	$response_code = wp_remote_retrieve_response_code($response);
4848		-
	2376	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' response code: ' . $response_code);
	2377	+
4849	2378	if ($response_code !== 200) {
	2379	+ $error_body = wp_remote_retrieve_body($response);
	2380	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' failed with body: ' . substr($error_body, 0, 500));
4850	2381	continue;
4851	2382	}
4852	2383
4853	2384	$body = wp_remote_retrieve_body($response);
		@@ -4853,8 +2384,9 @@
4853	2384	$body = wp_remote_retrieve_body($response);
4854	2385	$data = json_decode($body, true);
4855	2386
4856	2387	if (isset($data['matches'])) {
	2388	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' returned ' . count($data['matches']) . ' matches');
4857	2389	foreach ($data['matches'] as $match) {
4858	2390	$match_id = $match['id'] ?? '';
4859	2391	if (!empty($match_id) && !isset($seen_ids[$match_id])) {
4860	2392	$all_matches[] = $match;
		@@ -4860,14 +2392,18 @@
4860	2392	$all_matches[] = $match;
4861	2393	$seen_ids[$match_id] = true;
4862	2394	}
4863	2395	}
	2396	+ } else {
	2397	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' - no matches key in response');
4864	2398	}
4865	2399	}
4866	2400
4867		- // Convert matches to processed data format, grouping by URL to count chunks
	2401	+ //error_log('DEBUG: Total unique matches found: ' . count($all_matches));
	2402	+
	2403	+ // Convert matches to processed data format
4868	2404	$processed_data = array();
4869		- $url_chunk_counts = array();
	2405	+ $vector_ids_for_cache = array();
4870	2406
4871	2407	foreach ($all_matches as $match) {
4872	2408	$metadata = $match['metadata'] ?? array();
4873	2409	$source_url = $metadata['source_url'] ?? '';
		@@ -4875,14 +2411,8 @@
4875	2411
4876	2412	if (!empty($source_url) && !empty($match_id)) {
4877	2413	$post_id = url_to_postid($source_url);
4878	2414	if ($post_id) {
4879		- // Count chunks per post_id
4880		- if (!isset($url_chunk_counts[$post_id])) {
4881		- $url_chunk_counts[$post_id] = 0;
4882		- }
4883		- $url_chunk_counts[$post_id]++;
4884		-
4885	2415	$created_at = $metadata['created_at'] ?? '';
4886	2416	$processed_date = 'Recently';
4887	2417
4888	2418	if (!empty($created_at)) {
		@@ -4891,332 +2421,243 @@
4891	2421	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
4892	2422	}
4893	2423	}
4894	2424
4895		- // Only store if not already set, or update with newer timestamp
4896		- if (!isset($processed_data[$post_id]) ||
4897		- ($timestamp ?? 0) > ($processed_data[$post_id]['timestamp'] ?? 0)) {
4898		- $processed_data[$post_id] = array(
4899		- 'db_id' => $match_id,
4900		- 'processed_date' => $processed_date,
4901		- 'url' => $source_url,
4902		- 'source' => 'pinecone',
4903		- 'timestamp' => $timestamp ?? current_time('timestamp')
4904		- );
4905		- }
	2425	+ $processed_data[$post_id] = array(
	2426	+ 'db_id' => $match_id,
	2427	+ 'processed_date' => $processed_date,
	2428	+ 'url' => $source_url,
	2429	+ 'source' => 'pinecone',
	2430	+ 'timestamp' => $timestamp ?? current_time('timestamp')
	2431	+ );
	2432	+
	2433	+ $vector_ids_for_cache[] = $match_id;
4906	2434	}
4907	2435	}
4908	2436	}
4909	2437
4910		- // Add chunk counts to processed data
4911		- foreach ($url_chunk_counts as $post_id => $chunk_count) {
4912		- if (isset($processed_data[$post_id])) {
4913		- $processed_data[$post_id]['chunk_count'] = $chunk_count;
4914		- }
	2438	+ // Update the vector IDs cache for future use
	2439	+ if (!empty($vector_ids_for_cache)) {
	2440	+ update_option('mxchat_pinecone_vector_ids_cache', $vector_ids_for_cache);
	2441	+ //error_log('DEBUG: Updated vector IDs cache with ' . count($vector_ids_for_cache) . ' IDs');
4915	2442	}
4916	2443
	2444	+ //error_log('DEBUG: Returning ' . count($processed_data) . ' processed items from scanning');
4917	2445	return $processed_data;
4918	2446
4919	2447	} catch (Exception $e) {
	2448	+ //error_log('DEBUG: Exception in scan_pinecone_for_processed_content: ' . $e->getMessage());
4920	2449	return array();
4921	2450	}
4922	2451	}
4923		-/**
4924		- * Generate embeddings from input text for MXChat with bot support
4925		- */
4926		-private function mxchat_generate_embedding($text, $bot_id = 'default') {
4927		- // Enable detailed logging for debugging
4928		- //error_log('[MXCHAT-EMBED] Starting embedding generation for bot: ' . $bot_id . '. Text length: ' . strlen($text) . ' bytes');
4929		- //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4930	2452
4931		- // Get bot-specific options
4932		- $bot_options = $this->get_bot_options($bot_id);
4933		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	2453	+ /**
	2454	+ * Generates embeddings from input text for MXChat
	2455	+ */
	2456	+ private function mxchat_generate_embedding($text) {
	2457	+ // Enable detailed logging for debugging
	2458	+ //error_log('[MXCHAT-EMBED] Starting embedding generation. Text length: ' . strlen($text) . ' bytes');
	2459	+ //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4934	2460
4935		- // Opt-in: when the custom provider is selected for embeddings, index through
4936		- // the same custom endpoint the query path uses so stored vectors and query
4937		- // vectors share a model. Returns the vector array on success, or an error
4938		- // string on failure (this function's existing failure contract).
4939		- if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
4940		- if (!class_exists('MxChat_Utils')) {
4941		- require_once dirname(__FILE__) . '/../includes/class-mxchat-utils.php';
4942		- }
4943		- return MxChat_Utils::generate_embedding_custom($text, $options);
4944		- }
	2461	+ $options = get_option('mxchat_options');
	2462	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	2463	+ //error_log('[MXCHAT-EMBED] Selected embedding model: ' . $selected_model);
4945	2464
4946		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4947		- //error_log('[MXCHAT-EMBED] Selected embedding model for bot ' . $bot_id . ': ' . $selected_model);
	2465	+ // Determine provider and endpoint
	2466	+ if (strpos($selected_model, 'voyage') === 0) {
	2467	+ $api_key = $options['voyage_api_key'] ?? '';
	2468	+ $endpoint = 'https://api.voyageai.com/v1/embeddings';
	2469	+ $provider_name = 'Voyage AI';
	2470	+ //error_log('[MXCHAT-EMBED] Using Voyage AI API');
	2471	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	2472	+ $api_key = $options['gemini_api_key'] ?? '';
	2473	+ $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
	2474	+ $provider_name = 'Google Gemini';
	2475	+ //error_log('[MXCHAT-EMBED] Using Google Gemini API');
	2476	+ } else {
	2477	+ $api_key = $options['api_key'] ?? '';
	2478	+ $endpoint = 'https://api.openai.com/v1/embeddings';
	2479	+ $provider_name = 'OpenAI';
	2480	+ //error_log('[MXCHAT-EMBED] Using OpenAI API');
	2481	+ }
4948	2482
4949		- // Determine provider and endpoint
4950		- if (strpos($selected_model, 'voyage') === 0) {
4951		- $api_key = $options['voyage_api_key'] ?? '';
4952		- $endpoint = 'https://api.voyageai.com/v1/embeddings';
4953		- $provider_name = 'Voyage AI';
4954		- //error_log('[MXCHAT-EMBED] Using Voyage AI API for bot ' . $bot_id);
4955		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
4956		- $api_key = $options['gemini_api_key'] ?? '';
4957		- $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
4958		- $provider_name = 'Google Gemini';
4959		- //error_log('[MXCHAT-EMBED] Using Google Gemini API for bot ' . $bot_id);
4960		- } else {
4961		- $api_key = $options['api_key'] ?? '';
4962		- $endpoint = 'https://api.openai.com/v1/embeddings';
4963		- $provider_name = 'OpenAI';
4964		- //error_log('[MXCHAT-EMBED] Using OpenAI API for bot ' . $bot_id);
4965		- }
	2483	+ //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
4966	2484
4967		- //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
	2485	+ if (empty($api_key)) {
	2486	+ $error_message = sprintf('Missing %s API key. Please configure your API key in the MxChat settings.', $provider_name);
	2487	+ //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
	2488	+ return $error_message;
	2489	+ }
4968	2490
4969		- if (empty($api_key)) {
4970		- $error_message = sprintf('Missing %s API key for bot %s. Please configure your API key in the bot settings.', $provider_name, $bot_id);
4971		- //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
4972		- return $error_message;
4973		- }
	2491	+ // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
	2492	+ $estimated_tokens = ceil(str_word_count($text) / 0.75);
	2493	+ //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
4974	2494
4975		- // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
4976		- $estimated_tokens = ceil(str_word_count($text) / 0.75);
4977		- //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
	2495	+ if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
	2496	+ //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
	2497	+ // Consider truncating text here
	2498	+ }
4978	2499
4979		- if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
4980		- //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
4981		- // Consider truncating text here
4982		- }
	2500	+ // Prepare request body based on provider
	2501	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2502	+ // Gemini API format
	2503	+ $request_body = array(
	2504	+ 'model' => 'models/' . $selected_model,
	2505	+ 'content' => array(
	2506	+ 'parts' => array(
	2507	+ array('text' => $text)
	2508	+ )
	2509	+ )
	2510	+ );
4983	2511
4984		- // Prepare request body based on provider
4985		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4986		- // Gemini API format
4987		- $request_body = array(
4988		- 'model' => 'models/' . $selected_model,
4989		- 'content' => array(
4990		- 'parts' => array(
4991		- array('text' => $text)
4992		- )
4993		- )
4994		- );
	2512	+ // Set output dimensionality to 1536 for consistency with other models
	2513	+ $request_body['outputDimensionality'] = 1536;
	2514	+ } else {
	2515	+ // OpenAI/Voyage API format
	2516	+ $request_body = array(
	2517	+ 'model' => $selected_model,
	2518	+ 'input' => $text
	2519	+ );
4995	2520
4996		- // Set output dimensionality to 1536 for consistency with other models
4997		- $request_body['outputDimensionality'] = 1536;
4998		- } else {
4999		- // OpenAI/Voyage API format
5000		- $request_body = array(
5001		- 'model' => $selected_model,
5002		- 'input' => $text
5003		- );
	2521	+ // Add output_dimension for voyage-3-large model
	2522	+ if ($selected_model === 'voyage-3-large') {
	2523	+ $request_body['output_dimension'] = 2048;
	2524	+ }
	2525	+ }
5004	2526
5005		- // Add output_dimension for voyage-3-large model
5006		- if ($selected_model === 'voyage-3-large') {
5007		- $request_body['output_dimension'] = 2048;
5008		- }
5009		- }
	2527	+ //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
5010	2528
5011		- //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
	2529	+ // Prepare headers based on provider
	2530	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2531	+ // Gemini uses API key as query parameter
	2532	+ $endpoint .= '?key=' . $api_key;
	2533	+ $headers = array(
	2534	+ 'Content-Type' => 'application/json'
	2535	+ );
	2536	+ } else {
	2537	+ // OpenAI/Voyage use Bearer token
	2538	+ $headers = array(
	2539	+ 'Authorization' => 'Bearer ' . $api_key,
	2540	+ 'Content-Type' => 'application/json'
	2541	+ );
	2542	+ }
5012	2543
5013		- // Prepare headers based on provider
5014		- if (strpos($selected_model, 'gemini-embedding') === 0) {
5015		- // Gemini uses API key as query parameter
5016		- $endpoint .= '?key=' . $api_key;
5017		- $headers = array(
5018		- 'Content-Type' => 'application/json'
5019		- );
5020		- } else {
5021		- // OpenAI/Voyage use Bearer token
5022		- $headers = array(
5023		- 'Authorization' => 'Bearer ' . $api_key,
5024		- 'Content-Type' => 'application/json'
5025		- );
5026		- }
	2544	+ // Make API request
	2545	+ //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
	2546	+ $response = wp_remote_post($endpoint, array(
	2547	+ 'body' => wp_json_encode($request_body),
	2548	+ 'headers' => $headers,
	2549	+ 'timeout' => 60 // Increased timeout for large inputs
	2550	+ ));
5027	2551
5028		- // Make API request
5029		- //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
5030		- $response = wp_remote_post($endpoint, array(
5031		- 'body' => wp_json_encode($request_body),
5032		- 'headers' => $headers,
5033		- 'timeout' => 60 // Increased timeout for large inputs
5034		- ));
	2552	+ // Handle wp_remote_post errors
	2553	+ if (is_wp_error($response)) {
	2554	+ $error_message = $response->get_error_message();
	2555	+ //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
	2556	+ return 'Connection error: ' . $error_message;
	2557	+ }
5035	2558
5036		- // Handle wp_remote_post errors
5037		- if (is_wp_error($response)) {
5038		- $error_message = $response->get_error_message();
5039		- //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
5040		- return 'Connection error: ' . $error_message;
5041		- }
	2559	+ // Get and check HTTP response code
	2560	+ $http_code = wp_remote_retrieve_response_code($response);
	2561	+ //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
5042	2562
5043		- // Get and check HTTP response code
5044		- $http_code = wp_remote_retrieve_response_code($response);
5045		- //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
	2563	+ if ($http_code !== 200) {
	2564	+ $error_body = wp_remote_retrieve_body($response);
	2565	+ //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
5046	2566
5047		- if ($http_code !== 200) {
5048		- $error_body = wp_remote_retrieve_body($response);
5049		- //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
	2567	+ // Try to parse error for more details
	2568	+ $error_json = json_decode($error_body, true);
	2569	+ if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
	2570	+ $error_type = $error_json['error']['type'] ?? 'unknown';
	2571	+ $error_message = $error_json['error']['message'] ?? 'No message';
	2572	+ //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
	2573	+ //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
5050	2574
5051		- // Try to parse error for more details
5052		- $error_json = json_decode($error_body, true);
5053		- if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
5054		- $error_type = $error_json['error']['type'] ?? 'unknown';
5055		- $error_message = $error_json['error']['message'] ?? 'No message';
5056		- //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
5057		- //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
	2575	+ // Customize error message for common API errors
	2576	+ if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
	2577	+ $error_message = sprintf('Invalid %s API key. Please check your API key in the MxChat settings.', $provider_name);
	2578	+ } elseif ($error_type === 'authentication_error') {
	2579	+ $error_message = sprintf('%s authentication failed. Please verify your API key in the MxChat settings.', $provider_name);
	2580	+ }
5058	2581
5059		- // Keep the provider's own diagnostic — a restricted-key 401 names the
5060		- // exact missing scope, and replacing it with "check your API key" sent
5061		- // a customer to regenerate two keys (plan 46b596). Same shape as
5062		- // MxChat_Utils::embedding_failure_error() so both ingestion paths read
5063		- // identically. Key never appears in provider messages, but scrub anyway.
5064		- if ($error_type === 'invalid_request_error' || $error_type === 'authentication_error') {
5065		- if (is_string($api_key) && $api_key !== '') {
5066		- $error_message = str_replace($api_key, '[redacted]', $error_message);
5067		- }
5068		- $error_message = sprintf(
5069		- 'Embedding failed (%s, HTTP %d): %s',
5070		- $selected_model,
5071		- $http_code,
5072		- substr($error_message, 0, 300)
5073		- );
5074		- }
	2582	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2583	+ return $error_message;
	2584	+ }
5075	2585
5076		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
5077		- return $error_message;
5078		- }
	2586	+ $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding", $http_code);
	2587	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2588	+ return $error_message;
	2589	+ }
5079	2590
5080		- $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding for bot %s", $http_code, $bot_id);
5081		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
5082		- return $error_message;
5083		- }
	2591	+ // Parse response body
	2592	+ $response_body = wp_remote_retrieve_body($response);
	2593	+ //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
5084	2594
5085		- // Parse response body
5086		- $response_body = wp_remote_retrieve_body($response);
5087		- //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
	2595	+ $response_data = json_decode($response_body, true);
5088	2596
5089		- $response_data = json_decode($response_body, true);
	2597	+ if (json_last_error() !== JSON_ERROR_NONE) {
	2598	+ $error = json_last_error_msg();
	2599	+ //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
	2600	+ //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
	2601	+ return "Failed to parse API response: $error";
	2602	+ }
5090	2603
5091		- if (json_last_error() !== JSON_ERROR_NONE) {
5092		- $error = json_last_error_msg();
5093		- //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
5094		- //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
5095		- return "Failed to parse API response: $error";
5096		- }
	2604	+ // Handle different response formats based on provider
	2605	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2606	+ // Gemini API response format
	2607	+ if (isset($response_data['embedding']['values'])) {
	2608	+ $embedding_dimensions = count($response_data['embedding']['values']);
	2609	+ //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
5097	2610
5098		- // Handle different response formats based on provider
5099		- if (strpos($selected_model, 'gemini-embedding') === 0) {
5100		- // Gemini API response format
5101		- if (isset($response_data['embedding']['values'])) {
5102		- $embedding_dimensions = count($response_data['embedding']['values']);
5103		- //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
	2611	+ // Check if embedding dimensions are as expected (should be 1536)
	2612	+ if ($embedding_dimensions !== 1536) {
	2613	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
	2614	+ }
5104	2615
5105		- // Check if embedding dimensions are as expected (should be 1536)
5106		- if ($embedding_dimensions !== 1536) {
5107		- //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
5108		- }
	2616	+ return $response_data['embedding']['values'];
	2617	+ } else {
	2618	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
	2619	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
5109	2620
5110		- return $response_data['embedding']['values'];
5111		- } else {
5112		- //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
5113		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2621	+ if (isset($response_data['error'])) {
	2622	+ $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
	2623	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2624	+ return $error_message;
	2625	+ }
5114	2626
5115		- if (isset($response_data['error'])) {
5116		- $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
5117		- //error_log('[MXCHAT-EMBED] ' . $error_message);
5118		- return $error_message;
5119		- }
	2627	+ $error_message = "Invalid Gemini API response format: No embedding found";
	2628	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2629	+ return $error_message;
	2630	+ }
	2631	+ } else {
	2632	+ // OpenAI/Voyage API response format
	2633	+ if (isset($response_data['data'][0]['embedding'])) {
	2634	+ $embedding_dimensions = count($response_data['data'][0]['embedding']);
	2635	+ //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
5120	2636
5121		- $error_message = "Invalid Gemini API response format: No embedding found";
5122		- //error_log('[MXCHAT-EMBED] ' . $error_message);
5123		- return $error_message;
5124		- }
5125		- } else {
5126		- // OpenAI/Voyage API response format
5127		- if (isset($response_data['data'][0]['embedding'])) {
5128		- $embedding_dimensions = count($response_data['data'][0]['embedding']);
5129		- //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
	2637	+ // Check if embedding dimensions are as expected
	2638	+ if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
	2639	+ ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
	2640	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
	2641	+ }
5130	2642
5131		- // Check if embedding dimensions are as expected
5132		- if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
5133		- ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
5134		- //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
5135		- }
	2643	+ return $response_data['data'][0]['embedding'];
	2644	+ } else {
	2645	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
	2646	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
5136	2647
5137		- return $response_data['data'][0]['embedding'];
5138		- } else {
5139		- //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
5140		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2648	+ if (isset($response_data['error'])) {
	2649	+ $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
	2650	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2651	+ return $error_message;
	2652	+ }
5141	2653
5142		- if (isset($response_data['error'])) {
5143		- $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
5144		- //error_log('[MXCHAT-EMBED] ' . $error_message);
5145		- return $error_message;
5146		- }
5147		-
5148		- $error_message = "Invalid API response format: No embedding found";
5149		- //error_log('[MXCHAT-EMBED] ' . $error_message);
5150		- return $error_message;
5151		- }
5152		- }
5153		-}
5154		-
5155		-/**
5156		- * Get bot-specific options for multi-bot functionality
5157		- * Falls back to default options if bot_id is 'default' or multi-bot add-on is not active
5158		- */
5159		-private function get_bot_options($bot_id = 'default') {
5160		- //error_log("MXCHAT DEBUG: get_bot_options called for bot: " . $bot_id);
5161		-
5162		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
5163		- //error_log("MXCHAT DEBUG: Using default options (no multi-bot or bot is 'default')");
5164		- return array();
5165		- }
5166		-
5167		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
5168		-
5169		- if (!empty($bot_options)) {
5170		- //error_log("MXCHAT DEBUG: Got bot-specific options from filter");
5171		- if (isset($bot_options['similarity_threshold'])) {
5172		- //error_log(" - similarity_threshold: " . $bot_options['similarity_threshold']);
5173		- }
5174		- }
5175		-
5176		- return is_array($bot_options) ? $bot_options : array();
5177		-}
5178		-
5179		-/**
5180		- * Get bot-specific Pinecone configuration
5181		- * Used in the knowledge retrieval functions
5182		- */
5183		-// Also add debugging to your get_bot_pinecone_config function
5184		-private function get_bot_pinecone_config($bot_id = 'default') {
5185		- //error_log("MXCHAT DEBUG: get_bot_pinecone_config called for bot: " . $bot_id);
5186		-
5187		- // If default bot or multi-bot add-on not active, use default Pinecone config
5188		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
5189		- //error_log("MXCHAT DEBUG: Using default Pinecone config (no multi-bot or bot is 'default')");
5190		- $addon_options = get_option('mxchat_pinecone_addon_options', array());
5191		- $config = array(
5192		- 'use_pinecone' => (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1'),
5193		- 'api_key' => $addon_options['mxchat_pinecone_api_key'] ?? '',
5194		- 'host' => $addon_options['mxchat_pinecone_host'] ?? '',
5195		- 'namespace' => $addon_options['mxchat_pinecone_namespace'] ?? ''
5196		- );
5197		- //error_log("MXCHAT DEBUG: Default config - use_pinecone: " . ($config['use_pinecone'] ? 'true' : 'false'));
5198		- return $config;
5199		- }
5200		-
5201		- //error_log("MXCHAT DEBUG: Calling filter 'mxchat_get_bot_pinecone_config' for bot: " . $bot_id);
5202		-
5203		- // Hook for multi-bot add-on to provide bot-specific Pinecone config
5204		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
5205		-
5206		- if (!empty($bot_pinecone_config)) {
5207		- //error_log("MXCHAT DEBUG: Got bot-specific config from filter");
5208		- //error_log(" - use_pinecone: " . (isset($bot_pinecone_config['use_pinecone']) ? ($bot_pinecone_config['use_pinecone'] ? 'true' : 'false') : 'not set'));
5209		- //error_log(" - host: " . ($bot_pinecone_config['host'] ?? 'not set'));
5210		- //error_log(" - namespace: " . ($bot_pinecone_config['namespace'] ?? 'not set'));
5211		- } else {
5212		- //error_log("MXCHAT DEBUG: Filter returned empty config!");
5213		- }
5214		-
5215		- return is_array($bot_pinecone_config) ? $bot_pinecone_config : array();
5216		-}
5217		-
5218		-
	2654	+ $error_message = "Invalid API response format: No embedding found";
	2655	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2656	+ return $error_message;
	2657	+ }
	2658	+ }
	2659	+ }
5219	2660	public function mxchat_ajax_dismiss_completed_status() {
5220	2661	try {
5221	2662	// Verify the request
5222	2663	check_ajax_referer('mxchat_status_nonce', 'nonce');
		@@ -5579,593 +3020,18 @@
5579	3020
5580	3021	return $html;
5581	3022	}
5582	3023
5583		-/**
5584		- * Get all ACF fields for a specific post, excluding any fields the user has disabled
5585		- */
5586		-public function mxchat_get_acf_fields_for_post($post_id) {
5587		- if (!function_exists('get_field_objects')) {
5588		- return array();
5589		- }
5590		-
5591		- // Field OBJECTS, not get_fields(): exclusion matches on the field KEY
5592		- // (unique per field) rather than the name (shared across groups — plan
5593		- // 30e81f). ACF's own get_fields() is implemented as get_field_objects()
5594		- // reduced to name => value, so the un-excluded reduction below is the
5595		- // identical shape and order the previous get_fields() call produced.
5596		- $field_objects = get_field_objects($post_id);
5597		- if (!$field_objects || !is_array($field_objects)) {
5598		- return array();
5599		- }
5600		-
5601		- $excluded_fields = get_option('mxchat_acf_excluded_fields', array());
5602		- if (!is_array($excluded_fields)) {
5603		- $excluded_fields = array();
5604		- }
5605		-
5606		- $fields = array();
5607		- foreach ($field_objects as $field_name => $field_object) {
5608		- if (!empty($excluded_fields)) {
5609		- $field_key = isset($field_object['key']) ? $field_object['key'] : '';
5610		- // Legacy name entries stay honored: a stored name whose group was
5611		- // inactive at migration time still excludes every field wearing it.
5612		- if (in_array($field_key, $excluded_fields, true) || in_array($field_name, $excluded_fields, true)) {
5613		- continue;
5614		- }
5615		- }
5616		- $fields[$field_name] = isset($field_object['value']) ? $field_object['value'] : null;
5617		- }
5618		-
5619		- return $fields;
5620		-}
5621		-
5622		-/**
5623		- * Get all registered ACF field groups and their fields for the settings UI
5624		- */
5625		-public function mxchat_get_all_acf_fields() {
5626		- if (!function_exists('acf_get_field_groups') || !function_exists('acf_get_fields')) {
5627		- return array();
5628		- }
5629		-
5630		- // Keyed by GROUP KEY, not title (titles are not unique), and each field
5631		- // entry carries its ACF field key — the unique identifier every toggle,
5632		- // save, and index-time exclusion now runs on (plans 30e81f / bf57e0).
5633		- $all_fields = array();
5634		- $field_groups = acf_get_field_groups();
5635		-
5636		- if (!empty($field_groups)) {
5637		- foreach ($field_groups as $group) {
5638		- $group_fields = acf_get_fields($group['key']);
5639		- if (!empty($group_fields)) {
5640		- $entry = array(
5641		- 'title' => $group['title'],
5642		- 'fields' => array(),
5643		- );
5644		- foreach ($group_fields as $field) {
5645		- $entry['fields'][] = array(
5646		- 'key' => $field['key'],
5647		- 'name' => $field['name'],
5648		- 'label' => $field['label'],
5649		- 'type' => $field['type']
5650		- );
5651		- }
5652		- $all_fields[$group['key']] = $entry;
5653		- }
5654		- }
5655		- }
5656		-
5657		- return $all_fields;
5658		-}
5659		-
5660		-/**
5661		- * Get whitelisted custom post meta for a given post
5662		- * This allows non-ACF meta fields (like OptionTree, theme meta boxes) to be included in embeddings
5663		- */
5664		-public function mxchat_get_whitelisted_post_meta($post_id) {
5665		- $whitelist = get_option('mxchat_custom_meta_whitelist', '');
5666		-
5667		- if (empty($whitelist)) {
5668		- return array();
5669		- }
5670		-
5671		- // Parse the whitelist - one meta key per line
5672		- $meta_keys = array_filter(array_map('trim', explode("\n", $whitelist)));
5673		-
5674		- if (empty($meta_keys)) {
5675		- return array();
5676		- }
5677		-
5678		- $result = array();
5679		-
5680		- foreach ($meta_keys as $key) {
5681		- // Skip empty keys
5682		- if (empty($key)) {
5683		- continue;
5684		- }
5685		-
5686		- $value = get_post_meta($post_id, $key, true);
5687		-
5688		- // Only include non-empty string values
5689		- if (!empty($value) && is_string($value)) {
5690		- $result[$key] = $value;
5691		- } elseif (!empty($value) && is_array($value)) {
5692		- // Handle array values by joining them
5693		- $flat_value = $this->mxchat_flatten_meta_array($value);
5694		- if (!empty($flat_value)) {
5695		- $result[$key] = $flat_value;
5696		- }
5697		- }
5698		- }
5699		-
5700		- return $result;
5701		-}
5702		-
5703		-/**
5704		- * Flatten array meta values into a readable string
5705		- */
5706		-private function mxchat_flatten_meta_array($array, $depth = 0) {
5707		- if ($depth > 3) {
5708		- return ''; // Prevent infinite recursion
5709		- }
5710		-
5711		- $parts = array();
5712		-
5713		- foreach ($array as $key => $value) {
5714		- if (is_string($value) && !empty($value)) {
5715		- $parts[] = $value;
5716		- } elseif (is_array($value)) {
5717		- $nested = $this->mxchat_flatten_meta_array($value, $depth + 1);
5718		- if (!empty($nested)) {
5719		- $parts[] = $nested;
5720		- }
5721		- }
5722		- }
5723		-
5724		- return implode(', ', $parts);
5725		-}
5726		-
5727		-/**
5728		- * Format ACF field values for content extraction
5729		- */
5730		-public function mxchat_format_acf_field_value($value, $field_name = '', $post_id = 0) {
5731		- if (empty($value)) {
5732		- return '';
5733		- }
5734		-
5735		- // Handle WP_Post objects first (THIS IS THE KEY FIX)
5736		- if ($value instanceof WP_Post) {
5737		- return $value->post_title ?: '';
5738		- }
5739		-
5740		- // Handle other WP objects
5741		- if (is_object($value)) {
5742		- if (isset($value->post_title)) {
5743		- return $value->post_title;
5744		- } elseif (isset($value->display_name)) {
5745		- return $value->display_name;
5746		- } elseif (isset($value->name)) {
5747		- return $value->name;
5748		- } elseif (method_exists($value, '__toString')) {
5749		- try {
5750		- return (string) $value;
5751		- } catch (Exception $e) {
5752		- return '';
5753		- }
5754		- }
5755		- // For any other objects, return empty string
5756		- return '';
5757		- }
5758		-
5759		- // Handle different ACF field types
5760		- if (is_array($value)) {
5761		- // Check if it's an image/file field
5762		- if (isset($value['url'])) {
5763		- // Image field - return alt text, title, or caption
5764		- if (!empty($value['alt'])) {
5765		- return $value['alt'];
5766		- } elseif (!empty($value['title'])) {
5767		- return $value['title'];
5768		- } elseif (!empty($value['caption'])) {
5769		- return $value['caption'];
5770		- } else {
5771		- return ''; // Don't include just the URL
5772		- }
5773		- }
5774		-
5775		- // Check if it's a post object or relationship field
5776		- if (isset($value['post_title'])) {
5777		- return $value['post_title'];
5778		- }
5779		-
5780		- // Check if it's a user field
5781		- if (isset($value['display_name'])) {
5782		- return $value['display_name'];
5783		- }
5784		-
5785		- // Check if it's a taxonomy term
5786		- if (isset($value['name']) && isset($value['taxonomy'])) {
5787		- return $value['name'];
5788		- }
5789		-
5790		- // Check if it's a select field with label
5791		- if (isset($value['label'])) {
5792		- return $value['label'];
5793		- }
5794		-
5795		- // Check for repeater field or flexible content
5796		- if (is_numeric(key($value))) {
5797		- $sub_values = array();
5798		- foreach ($value as $sub_item) {
5799		- if (is_array($sub_item)) {
5800		- // For repeater/flexible content, extract text values
5801		- $sub_text = $this->mxchat_extract_text_from_acf_array($sub_item);
5802		- if (!empty($sub_text)) {
5803		- $sub_values[] = $sub_text;
5804		- }
5805		- } elseif ($sub_item instanceof WP_Post) {
5806		- // Handle WP_Post objects in arrays
5807		- $sub_values[] = $sub_item->post_title ?: '';
5808		- } else {
5809		- $sub_values[] = (string) $sub_item;
5810		- }
5811		- }
5812		- return implode(', ', array_filter($sub_values));
5813		- }
5814		-
5815		- // For other arrays, try to extract meaningful text
5816		- $text_values = array();
5817		- foreach ($value as $key => $val) {
5818		- if (is_string($val) && !empty(trim($val))) {
5819		- $text_values[] = trim($val);
5820		- } elseif ($val instanceof WP_Post) {
5821		- // Handle WP_Post objects in associative arrays
5822		- $text_values[] = $val->post_title ?: '';
5823		- } elseif (is_array($val) && isset($val['post_title'])) {
5824		- $text_values[] = $val['post_title'];
5825		- } elseif (is_array($val) && isset($val['name'])) {
5826		- $text_values[] = $val['name'];
5827		- }
5828		- }
5829		-
5830		- return implode(', ', array_filter($text_values));
5831		- }
5832		-
5833		- // Handle boolean values
5834		- if (is_bool($value)) {
5835		- return $value ? 'Yes' : 'No';
5836		- }
5837		-
5838		- // Handle numeric values
5839		- if (is_numeric($value)) {
5840		- return (string) $value;
5841		- }
5842		-
5843		- // Handle string values
5844		- if (is_string($value)) {
5845		- return trim($value);
5846		- }
5847		-
5848		- // For anything else that we can't handle, return empty string
5849		- // This prevents the "Object could not be converted to string" error
5850		- return '';
5851		-}
5852		-
5853		-/**
5854		- * Extract text from complex ACF array structures
5855		- */
5856		-private function mxchat_extract_text_from_acf_array($array) {
5857		- if (!is_array($array)) {
5858		- return '';
5859		- }
5860		-
5861		- $text_parts = array();
5862		-
5863		- foreach ($array as $key => $value) {
5864		- if (is_string($value) && !empty(trim($value))) {
5865		- // Skip keys that are likely to be IDs or technical values
5866		- if (!is_numeric($value) || strlen($value) > 10) {
5867		- $text_parts[] = trim($value);
5868		- }
5869		- } elseif ($value instanceof WP_Post) {
5870		- // Handle WP_Post objects
5871		- $text_parts[] = $value->post_title ?: '';
5872		- } elseif (is_array($value)) {
5873		- if (isset($value['post_title'])) {
5874		- $text_parts[] = $value['post_title'];
5875		- } elseif (isset($value['name'])) {
5876		- $text_parts[] = $value['name'];
5877		- } elseif (isset($value['label'])) {
5878		- $text_parts[] = $value['label'];
5879		- }
5880		- } elseif (is_object($value)) {
5881		- // Handle other objects safely
5882		- if (isset($value->post_title)) {
5883		- $text_parts[] = $value->post_title;
5884		- } elseif (isset($value->name)) {
5885		- $text_parts[] = $value->name;
5886		- } elseif (isset($value->display_name)) {
5887		- $text_parts[] = $value->display_name;
5888		- }
5889		- }
5890		- }
5891		-
5892		- return implode(', ', array_filter($text_parts));
5893		-}
5894		-
5895		-/**
5896		- * Walk an ACF field value tree and collect attachment IDs for any value that
5897		- * resolves to a PDF in the WordPress media library. Handles the three shapes
5898		- * ACF returns for File/Image/URL fields (array with ID+url, integer attachment ID,
5899		- * plain URL string), and recurses through repeater/group/flexible content.
5900		- *
5901		- * @param mixed $value The ACF field value (any depth)
5902		- * @param array $out Accumulator (passed by reference) for attachment IDs
5903		- * @param int $depth Recursion guard
5904		- */
5905		-private function mxchat_collect_pdf_attachment_ids_from_acf_value($value, &$out, $depth = 0) {
5906		- if ($depth > 6) {
5907		- return; // prevent runaway recursion on circular/very-deep structures
5908		- }
5909		-
5910		- if (empty($value)) {
5911		- return;
5912		- }
5913		-
5914		- // Array shapes: ACF File/Image return value=array; repeaters/groups are arrays of arrays
5915		- if (is_array($value)) {
5916		- // Direct File/Image-style array (has 'url' and usually 'ID' + 'mime_type')
5917		- $looks_like_attachment = isset($value['url']) || isset($value['ID']) || isset($value['id']);
5918		- if ($looks_like_attachment) {
5919		- $att_id = 0;
5920		- if (!empty($value['ID']) && is_numeric($value['ID'])) {
5921		- $att_id = (int) $value['ID'];
5922		- } elseif (!empty($value['id']) && is_numeric($value['id'])) {
5923		- $att_id = (int) $value['id'];
5924		- } elseif (!empty($value['url']) && is_string($value['url'])) {
5925		- $att_id = (int) attachment_url_to_postid($value['url']);
5926		- }
5927		-
5928		- $is_pdf = false;
5929		- if (!empty($value['mime_type']) && $value['mime_type'] === 'application/pdf') {
5930		- $is_pdf = true;
5931		- } elseif (!empty($value['subtype']) && strtolower((string) $value['subtype']) === 'pdf') {
5932		- $is_pdf = true;
5933		- } elseif (!empty($value['url']) && is_string($value['url']) && $this->mxchat_url_looks_like_pdf($value['url'])) {
5934		- $is_pdf = true;
5935		- } elseif ($att_id && get_post_mime_type($att_id) === 'application/pdf') {
5936		- $is_pdf = true;
5937		- }
5938		-
5939		- if ($is_pdf && $att_id && get_post_mime_type($att_id) === 'application/pdf') {
5940		- $out[] = $att_id;
5941		- }
5942		- // An array node that represents one attachment doesn't contain other
5943		- // attachments inside it — done with this branch.
5944		- return;
5945		- }
5946		-
5947		- // Recurse: repeater rows, flexible-content layouts, groups, etc.
5948		- foreach ($value as $sub) {
5949		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($sub, $out, $depth + 1);
5950		- }
5951		- return;
5952		- }
5953		-
5954		- // Plain numeric attachment ID (ACF File field set to "Return: ID")
5955		- if (is_numeric($value)) {
5956		- $att_id = (int) $value;
5957		- if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5958		- $out[] = $att_id;
5959		- }
5960		- return;
5961		- }
5962		-
5963		- // Plain string — URL pointing at a PDF (ACF File field set to "Return: URL", or a custom URL/text field)
5964		- if (is_string($value)) {
5965		- $trimmed = trim($value);
5966		- if ($trimmed !== '' && $this->mxchat_url_looks_like_pdf($trimmed)) {
5967		- $att_id = (int) attachment_url_to_postid($trimmed);
5968		- if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5969		- $out[] = $att_id;
5970		- }
5971		- }
5972		- return;
5973		- }
5974		-}
5975		-
5976		-/**
5977		- * Heuristic: does this URL/string look like a PDF reference?
5978		- * Tolerates query strings and fragments (#page=2).
5979		- */
5980		-private function mxchat_url_looks_like_pdf($url) {
5981		- if (!is_string($url) || $url === '') {
5982		- return false;
5983		- }
5984		- // Strip query + fragment before checking extension
5985		- $path = preg_replace('/[?#].*$/', '', $url);
5986		- return (bool) preg_match('/\.pdf$/i', $path);
5987		-}
5988		-
5989		-/**
5990		- * Extract text from a PDF attachment by ID using the bundled Smalot parser.
5991		- * Reads the file directly from disk via get_attached_file (no HTTP fetch).
5992		- * Result is cached on the attachment as post_meta keyed by file mtime so we
5993		- * only parse the same PDF once unless the file changes on disk.
5994		- *
5995		- * @param int $attachment_id
5996		- * @return string Extracted plain text, or '' on failure.
5997		- */
5998		-private function mxchat_extract_pdf_text_by_attachment_id($attachment_id) {
5999		- $attachment_id = (int) $attachment_id;
6000		- if ($attachment_id <= 0) {
6001		- return '';
6002		- }
6003		- if (get_post_mime_type($attachment_id) !== 'application/pdf') {
6004		- return '';
6005		- }
6006		-
6007		- $pdf_path = get_attached_file($attachment_id);
6008		- if (empty($pdf_path) || !file_exists($pdf_path) || !is_readable($pdf_path)) {
6009		- return '';
6010		- }
6011		-
6012		- // Raw-file size cap. Parsing very large PDFs can OOM the request; skip with a log entry
6013		- // and let the rest of the ACF content land in the KB. Filterable for users who need it bigger.
6014		- $default_max_bytes = 25 * 1024 * 1024;
6015		- $max_bytes = (int) apply_filters('mxchat_acf_pdf_max_bytes', $default_max_bytes, $attachment_id, $pdf_path);
6016		- if ($max_bytes > 0) {
6017		- $file_size = @filesize($pdf_path);
6018		- if ($file_size !== false && $file_size > $max_bytes) {
6019		- error_log(sprintf(
6020		- '[mxchat] ACF PDF skipped (over size cap): attachment %d "%s" %d bytes > cap %d',
6021		- $attachment_id,
6022		- basename($pdf_path),
6023		- $file_size,
6024		- $max_bytes
6025		- ));
6026		- return '';
6027		- }
6028		- }
6029		-
6030		- $mtime = @filemtime($pdf_path);
6031		- $cache_meta_key = '_mxchat_acf_pdf_text_v1';
6032		- $cached = get_post_meta($attachment_id, $cache_meta_key, true);
6033		- if (is_array($cached) && isset($cached['mtime'], $cached['text']) && (int) $cached['mtime'] === (int) $mtime) {
6034		- return (string) $cached['text'];
6035		- }
6036		-
6037		- $text = '';
6038		- try {
6039		- if (function_exists('mxchat_load_pdf_parser')) {
6040		- mxchat_load_pdf_parser();
6041		- }
6042		- if (!class_exists('\\Smalot\\PdfParser\\Parser')) {
6043		- return '';
6044		- }
6045		- $parser = new \Smalot\PdfParser\Parser();
6046		- $pdf = $parser->parseFile($pdf_path);
6047		- $pages = $pdf->getPages();
6048		- $page_texts = array();
6049		- $acf_page_num = 0;
6050		- foreach ($pages as $page) {
6051		- $acf_page_num++;
6052		- $page_text = '';
6053		- try {
6054		- $page_text = $page->getText();
6055		- } catch (\Exception $e) {
6056		- $page_text = '';
6057		- }
6058		- if (!empty($page_text)) {
6059		- $page_text = MxChat_Utils::normalize_pdf_rtl($page_text, 'acf_pdf attachment ' . $attachment_id . ' page ' . $acf_page_num);
6060		- $page_texts[] = $page_text;
6061		- }
6062		- }
6063		- $text = trim(implode("\n\n", $page_texts));
6064		- } catch (\Exception $e) {
6065		- error_log('[mxchat] ACF PDF extraction failed for attachment ' . $attachment_id . ': ' . $e->getMessage());
6066		- return '';
6067		- } catch (\Throwable $e) {
6068		- error_log('[mxchat] ACF PDF extraction error for attachment ' . $attachment_id . ': ' . $e->getMessage());
6069		- return '';
6070		- }
6071		-
6072		- // Cap per-PDF text to avoid blowing up the embedding payload on enormous PDFs.
6073		- // The chunker downstream will still split this into multiple vectors.
6074		- $max_len = (int) apply_filters('mxchat_acf_pdf_text_max_length', 50000);
6075		- if ($max_len > 0 && strlen($text) > $max_len) {
6076		- $text = substr($text, 0, $max_len);
6077		- }
6078		-
6079		- update_post_meta($attachment_id, $cache_meta_key, array(
6080		- 'mtime' => (int) $mtime,
6081		- 'text' => $text,
6082		- ));
6083		-
6084		- return $text;
6085		-}
6086		-
6087		-/**
6088		- * Handle ACF save - fires after ACF fields are saved
6089		- * This ensures ACF field data is available when syncing to knowledge base
6090		- */
6091		-public function mxchat_handle_acf_save($post_id) {
6092		- // Skip if not a valid post
6093		- if (!$post_id || $post_id === 'options') {
6094		- return;
6095		- }
6096		-
6097		- // Skip autosaves and revisions
	3024	+public function mxchat_handle_post_update($post_id, $post, $update) {
	3025	+ // Basic validation checks
6098	3026	if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
6099	3027	return;
6100	3028	}
6101		-
6102		- $post = get_post($post_id);
6103		- if (!$post) {
6104		- return;
6105		- }
6106		-
6107		- $post_type = $post->post_type;
6108		-
6109		- // Check if sync is enabled for this post type
6110		- $should_sync = false;
6111		-
6112		- if ($post_type === 'post' && get_option('mxchat_auto_sync_posts') === '1') {
6113		- $should_sync = true;
6114		- } else if ($post_type === 'page' && get_option('mxchat_auto_sync_pages') === '1') {
6115		- $should_sync = true;
6116		- } else if ($post_type === 'product' && class_exists('WooCommerce')) {
6117		- // WooCommerce products - check if WooCommerce integration is enabled
6118		- $options = get_option('mxchat_options', array());
6119		- if (isset($options['enable_woocommerce_integration']) &&
6120		- ($options['enable_woocommerce_integration'] === '1' || $options['enable_woocommerce_integration'] === 'on')) {
6121		- $should_sync = true;
6122		- }
6123		- } else {
6124		- // Check custom post types
6125		- $option_name = 'mxchat_auto_sync_' . $post_type;
6126		- if (get_option($option_name) === '1') {
6127		- $should_sync = true;
6128		- }
6129		- }
6130		-
6131		- if (!$should_sync) {
6132		- return;
6133		- }
6134		-
6135		- // Only process published posts
	3029	+
	3030	+ // Only process published content
6136	3031	if ($post->post_status !== 'publish') {
6137	3032	return;
6138	3033	}
6139		-
6140		- // Check if this post has any ACF fields - if not, no need to re-sync
6141		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
6142		- if (empty($acf_fields)) {
6143		- return;
6144		- }
6145		-
6146		- // Use a transient to prevent duplicate processing (post_updated may have already run)
6147		- $transient_key = 'mxchat_acf_synced_' . $post_id;
6148		- if (get_transient($transient_key)) {
6149		- return;
6150		- }
6151		- set_transient($transient_key, true, 60); // Prevent re-processing for 60 seconds
6152		-
6153		- // Re-run the sync with ACF data now available
6154		- // We pass $update=true since this is effectively an update with ACF data
6155		- $this->mxchat_handle_post_update($post_id, $post, true);
6156		-}
6157		-
6158		-public function mxchat_handle_post_update($post_id, $post, $update) {
6159		- // The in-flight-update marker has done its job the moment post_updated runs; drop it
6160		- // before any early return so it can never outlive its own save (a failed $wpdb->update
6161		- // inside wp_insert_post returns after pre_post_update but before the transition).
6162		- unset($this->pending_post_update[$post_id]);
6163		-
6164		- // Basic validation checks
6165		- if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
6166		- return;
6167		- }
6168	3034
6169	3035	$post_type = $post->post_type;
6170	3036
6171	3037	// Check if sync is enabled for this post type
		@@ -6187,263 +3053,29 @@
6187	3053	if (!$should_sync) {
6188	3054	return;
6189	3055	}
6190	3056
6191		- // Check if we have stored the previous status and URL in our transients
6192		- $previous_status_key = 'mxchat_prev_status_' . $post_id;
6193		- $previous_status = get_transient($previous_status_key);
	3057	+ // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
	3058	+ $title = get_the_title($post_id);
	3059	+ $content = get_post_field('post_content', $post_id);
6194	3060
6195		- $previous_url_key = 'mxchat_prev_url_' . $post_id;
6196		- $previous_url = get_transient($previous_url_key);
	3061	+ // Apply WordPress content filters to get properly formatted content
	3062	+ $content = apply_filters('the_content', $content);
6197	3063
6198		- // If the post was previously published but is now not published, remove from knowledge base
6199		- if ($previous_status === 'publish' && $post->post_status !== 'publish') {
6200		- // Use the stored URL from when it was published, or fall back to current permalink
6201		- $source_url = $previous_url ?: get_permalink($post_id);
6202		-
6203		- // mxchat_handle_status_transition already deleted for this post earlier in this
6204		- // request (it fires first inside wp_insert_post); skip the redundant round-trip.
6205		- if ($source_url && empty($this->transition_deleted_posts[$post_id])) {
6206		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
6207		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
6208		- }
6209		-
6210		- // Clean up the transients and exit early
6211		- delete_transient($previous_status_key);
6212		- delete_transient($previous_url_key);
6213		- return;
6214		- }
6215		-
6216		- // Slug/permalink rename while still published: delete the old vectors before upserting new ones.
6217		- // Without this, md5(old_url) vectors (base + chunks) would be orphaned under the stale URL.
6218		- if ($post->post_status === 'publish' && !empty($previous_url)) {
6219		- $current_url = get_permalink($post_id);
6220		- if ($current_url && $current_url !== $previous_url) {
6221		- MxChat_Utils::delete_chunks_for_url($previous_url, 'default');
6222		- }
6223		- }
6224		-
6225		- // Store the current status for next time (if this is an update)
6226		- if ($update) {
6227		- set_transient($previous_status_key, $post->post_status, DAY_IN_SECONDS);
6228		-
6229		- // If the post is currently published, also store its URL
6230		- if ($post->post_status === 'publish') {
6231		- $current_url = get_permalink($post_id);
6232		- set_transient($previous_url_key, $current_url, DAY_IN_SECONDS);
6233		- }
6234		- }
	3064	+ // Strip tags but preserve structure
	3065	+ $content = wp_strip_all_tags($content);
6235	3066
6236		- // Only process currently published content for adding/updating.
6237		- // transition_indexed_posts: mxchat_handle_status_transition's arrival edge may have
6238		- // already indexed this post earlier in this request (editor publishes fire
6239		- // transition_post_status first, then post_updated) — skip the duplicate embed.
6240		- // Consume-once: the flag is cleared when honoured, so a LATER save of the same
6241		- // post in one long-running process (WP-CLI scripts, importers) re-indexes normally.
6242		- if ($post->post_status === 'publish') {
6243		- if (!empty($this->transition_indexed_posts[$post_id])) {
6244		- unset($this->transition_indexed_posts[$post_id]);
6245		- } else {
6246		- $this->mxchat_index_published_post($post_id, $post);
6247		- }
6248		- }
6249		-
6250		- // Clean up the stored previous status if not used above
6251		- if ($previous_status !== 'publish' || $post->post_status === 'publish') {
6252		- delete_transient($previous_status_key);
6253		- delete_transient($previous_url_key);
6254		- }
6255		-}
6256		-
6257		-/**
6258		- * Index a published post into the knowledge base: preprocessing filter, content
6259		- * assembly (title/excerpt/body), WooCommerce product enrichment, job_listing meta,
6260		- * ACF fields (+ optional PDF extraction), whitelisted custom meta, embedding and
6261		- * upsert, then tag-based role restriction.
6262		- *
6263		- * Shared by the post_updated auto-sync path (mxchat_handle_post_update) and the
6264		- * transition_post_status arrival edge (mxchat_handle_status_transition), so
6265		- * scheduled publishes (wp_publish_post) and direct status=publish inserts index
6266		- * identically to editor saves (plan 3055e1). Pure extraction of the former
6267		- * publish branch — body indentation retained to keep the diff reviewable.
6268		- */
6269		-private function mxchat_index_published_post($post_id, $post) {
6270		- $post_type = $post->post_type;
6271		-
6272		- // WooCommerce products are owned by the WC-object assembler (plan a3d60c):
6273		- // whenever WooCommerce is active AND the integration is enabled, every
6274		- // product save also fires save_post_product, which queues
6275		- // mxchat_store_product_embedding on shutdown — and that writer runs LAST,
6276		- // overwriting the same md5(permalink) row this path would write. Assembling
6277		- // and embedding the product here was pure duplicate spend (measured: two
6278		- // embedding calls per product save, second one wins). Skip ONLY under the
6279		- // exact conditions the shutdown writer runs — same option read as its own
6280		- // gate — because with the integration off (or WooCommerce inactive) this
6281		- // path is the sole product indexer and must keep working.
6282		- if ($post_type === 'product'
6283		- && class_exists('WooCommerce')
6284		- && isset($this->options['enable_woocommerce_integration'])
6285		- && in_array($this->options['enable_woocommerce_integration'], ['1', 'on'])) {
6286		- return;
6287		- }
6288		-
6289		- // Get the source URL
6290		- $source_url = get_permalink($post_id);
6291		-
6292		- // A draft published programmatically (wp_publish_post) can reach this
6293		- // point with an EMPTY post_name — wp_insert_post skips slug generation
6294		- // for draft/pending — and get_permalink() then resolves to the bare
6295		- // site root. A knowledge row keyed to the homepage cites the wrong URL
6296		- // and answers homepage questions with this post's body, so refuse to
6297		- // write it; the post indexes correctly on its next save, once the slug
6298		- // exists. The empty-post_name test is what keeps a legitimate static
6299		- // front page (which has a slug but a root permalink) indexable.
6300		- // (Plan d138c4.)
6301		- if ('' === $post->post_name
6302		- && untrailingslashit($source_url) === untrailingslashit(home_url())) {
6303		- return;
6304		- }
6305		-
6306		- /**
6307		- * Allow developers to modify post data before processing into the knowledge base.
6308		- * Same filter and signature as the manual bulk-import path
6309		- * (ajax_mxchat_process_selected_content), so a callback registered once covers
6310		- * every indexing route. Purely additive — zero behaviour change when unhooked.
6311		- * Auto-sync runs under the 'default' bot context, matching the rest of this
6312		- * function.
6313		- *
6314		- * @param WP_Post $post The post about to be indexed.
6315		- * @param string $bot_id Bot context ('default' on auto-sync).
6316		- */
6317		- $post = apply_filters('mxchat_before_process_post', $post, 'default');
6318		- if (!($post instanceof WP_Post)) {
6319		- $post = get_post($post_id); // defend against a bad callback return
6320		- }
6321		-
6322		- // Assemble the indexable text via the shared post-kind assembler (a3d60c),
6323		- // reading from the FILTERED post object — not re-fetched by ID, which would
6324		- // discard it. Auto-sync reads content/excerpt in its historical
6325		- // get_post_field() display context, never appended product custom tabs
6326		- // (its product branch is reachable only with the WooCommerce integration
6327		- // off), and gates ACF→PDF extraction behind its own opt-in option —
6328		- // default OFF, because re-parsing every ACF PDF on every editor save is
6329		- // expensive and most sites don't want it (the 25 MB size cap lives in the
6330		- // shared extractor either way).
6331		- $prepared = $this->mxchat_prepare_post_content_for_indexing($post_id, $post, array(
6332		- 'read_display' => true,
6333		- 'extract_acf_pdfs' => get_option('mxchat_auto_sync_acf_pdfs', '0') === '1',
6334		- 'include_product_tabs' => false,
6335		- ));
6336		- $final_content = $prepared['content'];
6337		-
6338		- // Embedding decision — custom-provider-aware. Gating on a cloud API key
6339		- // here silently killed auto-sync on keyless custom-embeddings sites,
6340		- // because generate_embedding() routes custom FIRST and never needs the
6341		- // key (plan cbd5fd). Silent-return shape preserved.
6342		- $preflight = MxChat_Utils::embedding_preflight(get_option('mxchat_options'));
6343		- if (!$preflight['ok']) {
6344		- return;
6345		- }
6346		- $api_key = $preflight['api_key'];
6347		-
6348		- // Use the centralized utility function for storage
6349		- $result = MxChat_Utils::submit_content_to_db(
6350		- $final_content,
6351		- $source_url,
6352		- $api_key,
6353		- md5($source_url) // Vector ID for Pinecone
6354		- );
6355		-
6356		- // After successful storage, apply role restriction based on tags
6357		- if (!is_wp_error($result)) {
6358		- $this->apply_role_restriction_to_post($post_id, $source_url);
6359		- }
6360		-}
6361		-
6362		-/**
6363		- * Shared post-fields content assembler (plan a3d60c) — the ONE body behind both
6364		- * post-kind ingestion paths: manual bulk import (ajax_mxchat_process_selected_content)
6365		- * and auto-sync (mxchat_index_published_post). Behavior-preserving extraction; the
6366		- * measured per-caller differences ride $args instead of living as drifting copies:
6367		- *
6368		- * 'read_display' bool Auto-sync historically reads content/excerpt via
6369		- * get_post_field() in its default 'display' context
6370		- * (the post_content / post_excerpt display filters
6371		- * fire); bulk import reads the raw properties. Inert
6372		- * on a stock install — preserved per-path, not converged.
6373		- * 'extract_acf_pdfs' bool Each caller passes its OWN option (bulk:
6374		- * mxchat_acf_pdf_extraction; auto-sync:
6375		- * mxchat_auto_sync_acf_pdfs) — the two-option design
6376		- * is deliberate (plan 11720c). Gates BOTH the PDF-id
6377		- * collection walk and the extraction loop; the ids are
6378		- * only ever read inside the extraction branch, so
6379		- * gating collection is output-identical on every install.
6380		- * 'include_product_tabs' bool The bulk path has always appended yikes_woo custom
6381		- * tabs to product content; the auto-sync product branch
6382		- * (reachable only with the WooCommerce integration off)
6383		- * never did. Preserved per-path — converging it would be
6384		- * a behavior change, recorded on the plan instead.
6385		- *
6386		- * Returns array: 'content' (the assembled indexable text), 'acf_fields_found' and
6387		- * 'pdf_extracted_count' (the bulk path reports both in its AJAX response).
6388		- */
6389		-private function mxchat_prepare_post_content_for_indexing($post_id, $post, $args) {
6390		- $read_display = !empty($args['read_display']);
6391		- $extract_acf_pdfs = !empty($args['extract_acf_pdfs']);
6392		- $include_product_tabs = !empty($args['include_product_tabs']);
6393		-
6394		- // Raw post_title, NOT get_the_title(): the_title applies wptexturize +
6395		- // convert_chars and prepends the "Protected:" / "Private:" display chrome.
6396		- // The knowledge base stores facts, not display strings. Entity decode at
6397		- // output time (single-pass, shared helper) — a stored `&amp;` embeds worse
6398		- // than `&` and gets quoted back to visitors (d2c92e).
6399		- $content = $this->mxchat_decode_entities_for_indexing($post->post_title) . "\n\n";
6400		-
6401		- $raw_excerpt = $read_display ? get_post_field('post_excerpt', $post) : $post->post_excerpt;
6402		- $raw_content = $read_display ? get_post_field('post_content', $post) : $post->post_content;
6403		-
6404		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
6405		- // Strip FIRST, then test: an excerpt that is nothing but shortcodes strips to
6406		- // empty, and testing the raw value emitted a bare "Short Description: " label
6407		- // with no value after it. trim() only in the TEST — the emitted value is
6408		- // untouched, so a populated excerpt is byte-identical to before. A
6409		- // whitespace-only excerpt is an empty excerpt and must not produce a labelled
6410		- // line with nothing after it.
6411		- $clean_excerpt = $this->strip_shortcode_tags_preserve_content($raw_excerpt);
6412		- if (trim($clean_excerpt) !== '') {
6413		- $content .= "Short Description: " . $this->mxchat_decode_entities_for_indexing(wp_strip_all_tags($clean_excerpt)) . "\n\n";
6414		- }
6415		-
6416		- // Main content — remove shortcode tags but preserve content inside them, then
6417		- // strip tags (don't use 'the_content' filter as it may re-add shortcodes).
6418		- $clean_content = $this->strip_shortcode_tags_preserve_content($raw_content);
6419		- $content .= $this->mxchat_decode_entities_for_indexing(wp_strip_all_tags($clean_content));
6420		-
6421		- // WooCommerce product enrichment (post-fields kind). The WC-object assembler
6422		- // (mxchat_prepare_product_content_for_indexing) owns product rows whenever the
6423		- // integration is on; this branch serves the bulk import (all configurations)
6424		- // and auto-sync with the integration off.
6425		- if (get_post_type($post_id) === 'product' && class_exists('WooCommerce')) {
6426		- $product = wc_get_product($post_id);
6427		-
6428		- if ($product) {
6429		- $content .= "\n";
6430		- $content .= $this->mxchat_woo_product_summary_lines($product);
6431		- }
6432		-
6433		- if ($include_product_tabs) {
6434		- $content .= $this->mxchat_woo_custom_tabs_text($post_id);
6435		- }
6436		- }
6437		-
	3067	+ // Combine title and content
	3068	+ $final_content = $title . "\n\n" . $content;
	3069	+
6438	3070	// For custom post types like job_listing, include additional fields
6439		- if (get_post_type($post_id) === 'job_listing') {
	3071	+ if ($post_type === 'job_listing') {
6440	3072	// Add job-specific meta if available
6441	3073	$job_location = get_post_meta($post_id, '_job_location', true);
6442	3074	if (!empty($job_location)) {
6443		- $content .= "\n\nLocation: " . $job_location;
	3075	+ $final_content .= "\n\nLocation: " . $job_location;
6444	3076	}
6445		-
	3077	+
6446	3078	// Get job type terms
6447	3079	$job_types = get_the_terms($post_id, 'job_listing_type');
6448	3080	if (!empty($job_types) && !is_wp_error($job_types)) {
6449	3081	$types = array();
		@@ -6449,612 +3081,52 @@
6449	3081	$types = array();
6450	3082	foreach ($job_types as $type) {
6451	3083	$types[] = $type->name;
6452	3084	}
6453		- $content .= "\n\nJob Type: " . implode(', ', $types);
	3085	+ $final_content .= "\n\nJob Type: " . implode(', ', $types);
6454	3086	}
6455		-
	3087	+
6456	3088	// Get company name if available
6457	3089	$company_name = get_post_meta($post_id, '_company_name', true);
6458	3090	if (!empty($company_name)) {
6459		- $content .= "\n\nCompany: " . $company_name;
	3091	+ $final_content .= "\n\nCompany: " . $company_name;
6460	3092	}
6461	3093	}
6462		-
6463		- // ADD ACF FIELDS SUPPORT
6464		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
6465		- $pdf_extracted_count = 0;
6466		- if (!empty($acf_fields)) {
6467		- $acf_content_parts = array();
6468		- $pdf_attachment_ids = array();
6469		-
6470		- foreach ($acf_fields as $field_name => $field_value) {
6471		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
6472		-
6473		- if (!empty($formatted_value)) {
6474		- // Both separators: a hyphenated ACF name should read as words.
6475		- $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name));
6476		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
6477		- }
6478		-
6479		- // Walk this field's value tree for any PDF attachment references and
6480		- // queue them for extraction — only when this caller's PDF option is on.
6481		- if ($extract_acf_pdfs) {
6482		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
6483		- }
6484		- }
6485		-
6486		- if (!empty($acf_content_parts)) {
6487		- $content .= "\n\n" . implode("\n", $acf_content_parts);
6488		- }
6489		-
6490		- // Extract text from each unique PDF found in ACF fields and append as a labeled section
6491		- if ($extract_acf_pdfs && !empty($pdf_attachment_ids)) {
6492		- $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
6493		- $pdf_sections = array();
6494		- foreach ($pdf_attachment_ids as $att_id) {
6495		- $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
6496		- if (!empty($pdf_text)) {
6497		- $pdf_title = get_the_title($att_id);
6498		- $pdf_url = wp_get_attachment_url($att_id);
6499		- $header = 'PDF Attachment';
6500		- if (!empty($pdf_title)) {
6501		- $header .= ': ' . $pdf_title;
6502		- }
6503		- if (!empty($pdf_url)) {
6504		- $header .= ' (' . $pdf_url . ')';
6505		- }
6506		- $pdf_sections[] = $header . "\n" . $pdf_text;
6507		- $pdf_extracted_count++;
6508		- }
6509		- }
6510		- if (!empty($pdf_sections)) {
6511		- $content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
6512		- }
6513		- }
	3094	+
	3095	+ // Get the source URL
	3096	+ $source_url = get_permalink($post_id);
	3097	+
	3098	+ // Get API key with proper model detection
	3099	+ $options = get_option('mxchat_options');
	3100	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	3101	+
	3102	+ if (strpos($selected_model, 'voyage') === 0) {
	3103	+ $api_key = $options['voyage_api_key'] ?? '';
	3104	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	3105	+ $api_key = $options['gemini_api_key'] ?? '';
	3106	+ } else {
	3107	+ $api_key = $options['api_key'] ?? '';
6514	3108	}
6515		-
6516		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
6517		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
6518		- if (!empty($custom_meta)) {
6519		- $meta_content_parts = array();
6520		-
6521		- foreach ($custom_meta as $meta_key => $meta_value) {
6522		- // Convert meta key to readable label
6523		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
6524		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
6525		- }
6526		-
6527		- if (!empty($meta_content_parts)) {
6528		- $content .= "\n\n" . implode("\n", $meta_content_parts);
6529		- }
	3109	+
	3110	+ if (empty($api_key)) {
	3111	+ //error_log('MxChat Auto-sync: No API key configured for embedding model');
	3112	+ return;
6530	3113	}
6531		-
6532		- return array(
6533		- 'content' => $content,
6534		- 'acf_fields_found' => count($acf_fields),
6535		- 'pdf_extracted_count' => $pdf_extracted_count,
	3114	+
	3115	+ // Use the centralized utility function for storage
	3116	+ $result = MxChat_Utils::submit_content_to_db(
	3117	+ $final_content,
	3118	+ $source_url,
	3119	+ $api_key,
	3120	+ md5($source_url) // Vector ID for Pinecone
6536	3121	);
6537		-}
6538		-
6539		-/**
6540		- * Shared WC-object product assembler (plan a3d60c) — the ONE body behind the two
6541		- * WooCommerce-object ingestion paths: the auto-sync product writer
6542		- * (mxchat_store_product_embedding) and the URL/sitemap product import
6543		- * (mxchat_extract_woocommerce_product_content). Assembles from the WC_Product,
6544		- * the authoritative source for product rows (scope decision on the plan).
6545		- */
6546		-private function mxchat_prepare_product_content_for_indexing($product) {
6547		- $title = $product->get_name();
6548		- $description = $product->get_description();
6549		- $short_description = $product->get_short_description();
6550		-
6551		- // Format content consistently
6552		- $content = $title . "\n\n";
6553		-
6554		- if (!empty($short_description)) {
6555		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
6556		- }
6557		-
6558		- if (!empty($description)) {
6559		- $content .= wp_strip_all_tags($description) . "\n\n";
6560		- }
6561		-
6562		- $content .= $this->mxchat_woo_product_summary_lines($product);
6563		- $content .= $this->mxchat_woo_custom_tabs_text($product->get_id());
6564		-
6565		- return $content;
6566		-}
6567		-
6568		-/**
6569		- * Pricing + SKU + categories lines for a product — shared by both assembler kinds
6570		- * (the post-fields product enrichment and the WC-object assembler).
6571		- */
6572		-private function mxchat_woo_product_summary_lines($product) {
6573		- // Add pricing information (base-currency pinned — see mxchat_product_price_lines)
6574		- $lines = $this->mxchat_product_price_lines($product);
6575		-
6576		- $sku = $product->get_sku();
6577		- if (!empty($sku)) {
6578		- $lines .= "SKU: " . $sku . "\n";
6579		- }
6580		-
6581		- // Get product categories
6582		- $categories = wp_get_post_terms($product->get_id(), 'product_cat', array('fields' => 'names'));
6583		- if (!empty($categories) && !is_wp_error($categories)) {
6584		- $lines .= "Categories: " . implode(', ', $categories) . "\n";
6585		- }
6586		-
6587		- return $lines;
6588		-}
6589		-
6590		-/**
6591		- * Custom Product Tabs text (supports "Custom Product Tabs for WooCommerce" by
6592		- * Code Parrots) — direct tabs plus applied reusable/saved tabs. The ONE copy of
6593		- * the yikes_woo logic; three sites carried byte-identical clones before a3d60c.
6594		- */
6595		-private function mxchat_woo_custom_tabs_text($product_id) {
6596		- $text = '';
6597		-
6598		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
6599		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
6600		- foreach ($custom_tabs as $tab) {
6601		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
6602		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
6603		-
6604		- if (!empty($tab_title) && !empty($tab_content)) {
6605		- $text .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
6606		- }
6607		- }
6608		- }
6609		-
6610		- // Also check for reusable/saved tabs applied to this product
6611		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
6612		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
6613		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
6614		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
6615		- foreach ($applied_saved_tabs as $saved_tab_id) {
6616		- if (isset($saved_tabs[$saved_tab_id])) {
6617		- $tab = $saved_tabs[$saved_tab_id];
6618		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
6619		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
6620		-
6621		- if (!empty($tab_title) && !empty($tab_content)) {
6622		- $text .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
6623		- }
6624		- }
6625		- }
6626		- }
6627		- }
6628		-
6629		- return $text;
6630		-}
6631		-
6632		-/**
6633		- * Store the post status and URL before update to detect status transitions
6634		- * This runs before the post is actually updated in the database
6635		- */
6636		-public function mxchat_store_pre_update_status($post_id, $data) {
6637		- // Core is inside wp_insert_post's update branch, so a post_updated WILL fire later
6638		- // this request and can consume the arrival-edge guard (plan a664f3).
6639		- $this->pending_post_update[$post_id] = true;
6640		-
6641		- // Get the current post from database (before update)
6642		- $current_post = get_post($post_id);
6643	3122
6644		- if ($current_post) {
6645		- // Store the current status temporarily
6646		- $status_key = 'mxchat_prev_status_' . $post_id;
6647		- set_transient($status_key, $current_post->post_status, HOUR_IN_SECONDS);
6648		-
6649		- // If the post is currently published, also store its URL
6650		- if ($current_post->post_status === 'publish') {
6651		- $url_key = 'mxchat_prev_url_' . $post_id;
6652		- $current_url = get_permalink($post_id);
6653		- set_transient($url_key, $current_url, HOUR_IN_SECONDS);
6654		- }
	3123	+ if (is_wp_error($result)) {
	3124	+ //error_log('MxChat Auto-sync failed for post ' . $post_id . ': ' . $result->get_error_message());
6655	3125	}
6656	3126	}
6657	3127
6658		-/**
6659		- * Whether auto-sync is enabled for a post type (mirrors the checks used by the
6660		- * update/delete handlers; kept as one helper so new call sites cannot drift).
6661		- */
6662		-private function mxchat_is_auto_sync_enabled($post_type) {
6663		- if ($post_type === 'post') {
6664		- return get_option('mxchat_auto_sync_posts') === '1';
6665		- }
6666		- if ($post_type === 'page') {
6667		- return get_option('mxchat_auto_sync_pages') === '1';
6668		- }
6669		- return get_option('mxchat_auto_sync_' . $post_type) === '1';
6670		-}
6671	3128
6672		-/**
6673		- * Remove a post's vectors the moment it leaves 'publish', using the authoritative
6674		- * old status core passes to transition_post_status — no transient involved (plan 816fb1).
6675		- *
6676		- * Covers status changes that never route through wp_update_post (scheduled-expiry
6677		- * plugins and others that flip post_status directly and call wp_transition_post_status),
6678		- * where neither pre_post_update nor post_updated fires and the old detection missed.
6679		- */
6680		-public function mxchat_handle_status_transition($new_status, $old_status, $post) {
6681		- if (!($post instanceof WP_Post) || wp_is_post_revision($post->ID)) {
6682		- return;
6683		- }
6684		-
6685		- // Arrival edge (plan 3055e1): a post BECOMING published is indexed here, because
6686		- // wp_publish_post() — the path scheduled posts take via check_and_publish_future_post —
6687		- // and direct wp_insert_post(status=publish) creates never fire post_updated, so the
6688		- // auto-sync ADD path alone misses them. Editor publishes also pass through here;
6689		- // the transition_indexed_posts guard keeps mxchat_handle_post_update from embedding
6690		- // a second time in the same request.
6691		- if ($new_status === 'publish' && $old_status !== 'publish') {
6692		- if ($this->mxchat_is_auto_sync_enabled($post->post_type)) {
6693		- $this->mxchat_index_published_post($post->ID, $post);
6694		-
6695		- // Arm the double-fire guard ONLY when a post_updated is actually coming to
6696		- // consume it (plan a664f3). Two publish paths never fire post_updated at all:
6697		- // a direct wp_insert_post(status=publish) create, and wp_publish_post() — the
6698		- // call check_and_publish_future_post() makes for scheduled posts. Arming the
6699		- // guard unconditionally left it set with nothing to consume it, so the NEXT
6700		- // update of that post was swallowed entirely: zero embed calls, no knowledge
6701		- // -base row, silently. Consume-once on this side too, so a guard can never
6702		- // outlive the single save it was armed for.
6703		- if (!empty($this->pending_post_update[$post->ID])) {
6704		- unset($this->pending_post_update[$post->ID]);
6705		- $this->transition_indexed_posts[$post->ID] = true;
6706		- }
6707		- }
6708		- return;
6709		- }
6710		-
6711		- // Only the publish -> not-publish edge matters here.
6712		- if ($old_status !== 'publish' || $new_status === 'publish') {
6713		- return;
6714		- }
6715		- // Trash is handled by mxchat_handle_post_delete (wp_trash_post) with pre-trash URL
6716		- // resolution; skip to avoid a second network round-trip per trash.
6717		- if ($new_status === 'trash') {
6718		- return;
6719		- }
6720		- if (!$this->mxchat_is_auto_sync_enabled($post->post_type)) {
6721		- return;
6722		- }
6723		-
6724		- $urls = array();
6725		-
6726		- // The DB may already hold the new status when this fires, so get_permalink() on the
6727		- // live post could build a draft-style URL whose md5 misses the stored vector IDs.
6728		- // Reconstruct the published permalink from a clone instead.
6729		- $published_clone = clone $post;
6730		- $published_clone->post_status = 'publish';
6731		- $published_url = get_permalink($published_clone);
6732		- if ($published_url) {
6733		- $urls[] = $published_url;
6734		- }
6735		-
6736		- // Honour the pre-update capture when present (covers a slug change in the same save).
6737		- $previous_url = get_transient('mxchat_prev_url_' . $post->ID);
6738		- if (!empty($previous_url)) {
6739		- $urls[] = $previous_url;
6740		- }
6741		-
6742		- foreach (array_unique($urls) as $url) {
6743		- MxChat_Utils::delete_chunks_for_url($url, 'default');
6744		- }
6745		-
6746		- if (!empty($urls)) {
6747		- $this->transition_deleted_posts[$post->ID] = true;
6748		- }
6749		-}
6750		-
6751		-/**
6752		- * WP-CLI: remove knowledge-base entries left behind by posts that were unpublished,
6753		- * trashed, or made private before the transition_post_status handler existed.
6754		- *
6755		- * Walks every auto-synced post type's non-published posts, reconstructs each one's
6756		- * published-era permalink, and deletes its vectors (routes to Pinecone or the WP table).
6757		- * Deletion is idempotent, so never-indexed posts are a cheap no-op.
6758		- *
6759		- * ## OPTIONS
6760		- *
6761		- * [--dry-run]
6762		- * : Report what would be removed without deleting anything.
6763		- *
6764		- * ## EXAMPLES
6765		- *
6766		- * wp mxchat prune-unpublished --dry-run
6767		- * wp mxchat prune-unpublished
6768		- */
6769		-public function cli_prune_unpublished($args, $assoc_args) {
6770		- global $wpdb;
6771		- $dry_run = !empty($assoc_args['dry-run']);
6772		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
6773		-
6774		- $candidate_types = array_merge(array('post', 'page'), array_values(get_post_types(array('_builtin' => false), 'names')));
6775		- $synced_types = array();
6776		- foreach ($candidate_types as $type) {
6777		- if ($this->mxchat_is_auto_sync_enabled($type)) {
6778		- $synced_types[] = $type;
6779		- }
6780		- }
6781		- if (empty($synced_types)) {
6782		- WP_CLI::success('No post types have auto-sync enabled; nothing to prune.');
6783		- return;
6784		- }
6785		-
6786		- $scanned = 0;
6787		- $pruned = 0;
6788		- $paged = 1;
6789		- do {
6790		- $query = new WP_Query(array(
6791		- 'post_type' => $synced_types,
6792		- 'post_status' => array('draft', 'pending', 'private', 'future', 'trash'),
6793		- 'posts_per_page' => 100,
6794		- 'paged' => $paged,
6795		- 'fields' => 'ids',
6796		- ));
6797		- foreach ($query->posts as $post_id) {
6798		- $post = get_post($post_id);
6799		- if (!$post) {
6800		- continue;
6801		- }
6802		- $scanned++;
6803		-
6804		- // Rebuild the permalink the post had while published: publish-status clone,
6805		- // with wp_trash_post's __trashed slug suffix stripped for trashed posts.
6806		- $clone = clone $post;
6807		- $clone->post_status = 'publish';
6808		- if (substr($clone->post_name, -9) === '__trashed') {
6809		- $clone->post_name = substr($clone->post_name, 0, -9);
6810		- }
6811		- $url = get_permalink($clone);
6812		- if (!$url) {
6813		- continue;
6814		- }
6815		-
6816		- // Local-table row count is exact in WordPress-DB mode; in Pinecone mode it
6817		- // reads 0 but the delete below still routes to Pinecone and is idempotent.
6818		- $local_rows = (int) $wpdb->get_var($wpdb->prepare(
6819		- "SELECT COUNT(*) FROM {$table} WHERE source_url = %s", $url
6820		- ));
6821		-
6822		- if ($dry_run) {
6823		- if ($local_rows > 0) {
6824		- WP_CLI::log(sprintf('Would remove %d row(s): %s (post %d, %s)', $local_rows, $url, $post_id, $post->post_status));
6825		- $pruned += $local_rows;
6826		- }
6827		- continue;
6828		- }
6829		-
6830		- MxChat_Utils::delete_chunks_for_url($url, 'default');
6831		- if ($local_rows > 0) {
6832		- WP_CLI::log(sprintf('Removed %d row(s): %s (post %d, %s)', $local_rows, $url, $post_id, $post->post_status));
6833		- $pruned += $local_rows;
6834		- }
6835		- }
6836		- $more = $paged < $query->max_num_pages;
6837		- $paged++;
6838		- } while ($more);
6839		-
6840		- WP_CLI::success(sprintf(
6841		- '%s %d local knowledge row(s) across %d non-published post(s) scanned.%s',
6842		- $dry_run ? 'Would remove' : 'Removed',
6843		- $pruned,
6844		- $scanned,
6845		- ' (Pinecone-mode deletions are not counted locally.)'
6846		- ));
6847		-}
6848		-
6849		-/**
6850		- * WP-CLI: repair knowledge-base rows whose PDF text was imported in visual
6851		- * (reversed) order before the RTL normalizer existed. 32bf9e fixed new
6852		- * imports only; this fixes rows already in the table without the customer
6853		- * having to re-source and re-upload the original PDFs (plan d1e6f7).
6854		- *
6855		- * Detection reuses MxChat_Utils::normalize_pdf_rtl() on the stored text: a
6856		- * row is a candidate exactly when the normalizer would change it, so the
6857		- * import-time heuristic and the repair heuristic can never disagree.
6858		- * Repaired rows are RE-EMBEDDED — the stored vector was computed over
6859		- * reversed text and is as broken as the text — so a wet run calls the
6860		- * embedding provider once per repaired row on the site's API key. Runs
6861		- * beyond 25 rows therefore require --yes.
6862		- *
6863		- * Scope notes:
6864		- * - Scans the WordPress knowledge table. Pinecone-mode entries live in
6865		- * Pinecone, not this table, and are not scanned; if a scanned row's bot
6866		- * ALSO has Pinecone enabled (hybrid drift), the repaired entry is
6867		- * re-submitted through the normal import path so the md5-keyed Pinecone
6868		- * vector is replaced too.
6869		- * - Knowledge rows do not carry a bot id; --bot only selects whose
6870		- * embedding configuration (model + key) is used for re-embedding.
6871		- * - The mxchat_pdf_rtl_normalize filter is honoured: a site that disabled
6872		- * normalization gets detections of zero, not surprise rewrites.
6873		- * - The metadata header the PDF importer stores before the text separator
6874		- * is preserved byte-identical; only the text segment is repaired.
6875		- *
6876		- * ## OPTIONS
6877		- *
6878		- * [--dry-run]
6879		- * : List the rows that would be repaired without changing anything.
6880		- *
6881		- * [--bot=<id>]
6882		- * : Embedding configuration to use for re-embedding. Default: default.
6883		- *
6884		- * [--all-content]
6885		- * : Scan every row containing right-to-left text, not just rows with PDF
6886		- * provenance (a page anchor in the source URL, or pdf content type).
6887		- *
6888		- * [--yes]
6889		- * : Proceed even when more than 25 rows need re-embedding (API cost gate).
6890		- *
6891		- * ## EXAMPLES
6892		- *
6893		- * wp mxchat rtl-repair --dry-run
6894		- * wp mxchat rtl-repair
6895		- * wp mxchat rtl-repair --all-content --yes
6896		- */
6897		-public function cli_rtl_repair($args, $assoc_args) {
6898		- global $wpdb;
6899		- $dry_run = !empty($assoc_args['dry-run']);
6900		- $all = !empty($assoc_args['all-content']);
6901		- $yes = !empty($assoc_args['yes']);
6902		- $bot_id = isset($assoc_args['bot']) ? sanitize_key($assoc_args['bot']) : 'default';
6903		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
6904		-
6905		- // Detection pass — no API calls. Walk the table in id batches so a large
6906		- // knowledge base never loads at once.
6907		- $rtl_re = '/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u';
6908		- $candidates = array();
6909		- $scanned = 0;
6910		- $last_id = 0;
6911		- do {
6912		- if ($all) {
6913		- $rows = $wpdb->get_results($wpdb->prepare(
6914		- "SELECT id, article_content, source_url, content_type FROM {$table}
6915		- WHERE id > %d ORDER BY id ASC LIMIT 200",
6916		- $last_id
6917		- ));
6918		- } else {
6919		- $rows = $wpdb->get_results($wpdb->prepare(
6920		- "SELECT id, article_content, source_url, content_type FROM {$table}
6921		- WHERE id > %d AND (source_url LIKE %s OR content_type = 'pdf')
6922		- ORDER BY id ASC LIMIT 200",
6923		- $last_id,
6924		- '%' . $wpdb->esc_like('#page=') . '%'
6925		- ));
6926		- }
6927		- foreach ($rows as $row) {
6928		- $last_id = (int) $row->id;
6929		- $scanned++;
6930		- $content = (string) $row->article_content;
6931		- if (!preg_match($rtl_re, $content)) {
6932		- continue;
6933		- }
6934		- list($header, $text) = $this->mxchat_rtl_repair_split($content);
6935		- $normalized = MxChat_Utils::normalize_pdf_rtl($text, 'rtl-repair row ' . $row->id);
6936		- if (is_string($normalized) && $normalized !== $text) {
6937		- $candidates[] = array(
6938		- 'id' => (int) $row->id,
6939		- 'source_url' => (string) $row->source_url,
6940		- 'content_type' => (string) $row->content_type,
6941		- 'new_content' => $header . $normalized,
6942		- );
6943		- }
6944		- }
6945		- } while (count($rows) === 200);
6946		-
6947		- WP_CLI::log(sprintf('Scanned %d row(s); %d stored in reversed (visual) order.', $scanned, count($candidates)));
6948		- if (empty($candidates)) {
6949		- WP_CLI::success('No reversed RTL rows found — nothing to repair.');
6950		- return;
6951		- }
6952		-
6953		- foreach ($candidates as $c) {
6954		- WP_CLI::log(sprintf('%s row %d %s', $dry_run ? 'Would repair' : 'Will repair', $c['id'], $c['source_url']));
6955		- }
6956		- if ($dry_run) {
6957		- WP_CLI::success(sprintf('Dry run: %d row(s) would be repaired and re-embedded. Run without --dry-run to apply.', count($candidates)));
6958		- return;
6959		- }
6960		-
6961		- // Cost gate: re-embedding spends the customer's API budget.
6962		- WP_CLI::log(sprintf('Re-embedding will call the embedding provider once per row — %d call(s) on this site\'s API key.', count($candidates)));
6963		- if (count($candidates) > 25 && !$yes) {
6964		- WP_CLI::error(sprintf('%d rows need re-embedding (more than 25). Re-run with --yes to confirm the API cost. No rows were changed.', count($candidates)));
6965		- }
6966		-
6967		- $bot_options = $this->get_bot_options($bot_id);
6968		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
6969		- $preflight = MxChat_Utils::embedding_preflight($options);
6970		- if (!$preflight['ok']) {
6971		- WP_CLI::error('Embedding configuration problem: ' . $preflight['reason']);
6972		- }
6973		- $api_key = $preflight['api_key'];
6974		-
6975		- $pinecone_hybrid = $this->mxchat_rtl_repair_pinecone_enabled($bot_id);
6976		- $repaired = 0;
6977		- $failed = 0;
6978		- foreach ($candidates as $c) {
6979		- $vector = MxChat_Utils::regenerate_embedding($c['new_content'], $api_key, $bot_id);
6980		- if (!is_array($vector)) {
6981		- $failed++;
6982		- $reason = is_wp_error($vector) ? $vector->get_error_message() : 'embedding request failed';
6983		- // Text and vector must stay consistent: never write repaired text
6984		- // beside the stale reversed-text vector.
6985		- WP_CLI::warning(sprintf('Row %d NOT repaired — %s. Row left unchanged.', $c['id'], $reason));
6986		- continue;
6987		- }
6988		- $wpdb->update(
6989		- $table,
6990		- array(
6991		- 'article_content' => $c['new_content'],
6992		- 'embedding_vector' => maybe_serialize($vector),
6993		- ),
6994		- array('id' => $c['id']),
6995		- array('%s', '%s'),
6996		- array('%d')
6997		- );
6998		- $repaired++;
6999		- if (class_exists('MxChat_Admin')) {
7000		- MxChat_Admin::mxchat_log_debug('pdf_rtl_repaired', 'Stored KB row restored to logical order and re-embedded', array(
7001		- 'row_id' => $c['id'],
7002		- 'source_url' => $c['source_url'],
7003		- 'bot' => $bot_id,
7004		- ));
7005		- }
7006		- // Hybrid drift: the bot indexes into Pinecone but this row sat in the
7007		- // WP table — push the repaired entry through the normal import path so
7008		- // the md5(source_url)-keyed Pinecone vector is replaced as well.
7009		- if ($pinecone_hybrid) {
7010		- MxChat_Utils::submit_content_to_db(
7011		- $c['new_content'],
7012		- $c['source_url'],
7013		- $api_key,
7014		- null,
7015		- $bot_id,
7016		- $c['content_type'] !== '' ? $c['content_type'] : 'pdf'
7017		- );
7018		- }
7019		- }
7020		-
7021		- WP_CLI::success(sprintf('Repaired + re-embedded %d row(s); %d failed; %d scanned.', $repaired, $failed, $scanned));
7022		-}
7023		-
7024		-/**
7025		- * Split a stored KB row into (metadata header incl. separator, text segment).
7026		- * The PDF importer stores wp_json_encode($metadata) . "\n---\n" . $text —
7027		- * repair must touch only the text and keep the header byte-identical.
7028		- */
7029		-private function mxchat_rtl_repair_split($content) {
7030		- $sep = "\n---\n";
7031		- $pos = strpos($content, $sep);
7032		- if ($pos !== false && $pos > 0 && $content[0] === '{') {
7033		- $maybe_json = substr($content, 0, $pos);
7034		- if (json_decode($maybe_json) !== null) {
7035		- return array(substr($content, 0, $pos + strlen($sep)), substr($content, $pos + strlen($sep)));
7036		- }
7037		- }
7038		- return array('', $content);
7039		-}
7040		-
7041		-/**
7042		- * Mirror of MxChat_Utils::is_pinecone_enabled_for_bot() (private there) for
7043		- * the repair CLI's hybrid-drift check.
7044		- */
7045		-private function mxchat_rtl_repair_pinecone_enabled($bot_id) {
7046		- if ($bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager')) {
7047		- $cfg = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
7048		- if (!empty($cfg)) {
7049		- return !empty($cfg['use_pinecone']) && !empty($cfg['api_key']) && !empty($cfg['host']);
7050		- }
7051		- }
7052		- $po = get_option('mxchat_pinecone_addon_options');
7053		- return !empty($po['mxchat_use_pinecone']) && $po['mxchat_use_pinecone'] !== '0'
7054		- && !empty($po['mxchat_pinecone_api_key']) && !empty($po['mxchat_pinecone_host']);
7055		-}
7056		-
7057	3129	public function mxchat_handle_post_delete($post_id) {
7058	3130	// Get post data before it's deleted
7059	3131	$post = get_post($post_id);
7060	3132
		@@ -7084,51 +3156,85 @@
7084	3156	if (!$should_sync) {
7085	3157	return;
7086	3158	}
7087	3159
7088		- // Resolve the pre-trash URL. wp_trash_post renames the slug with "__trashed" before firing
7089		- // this hook, so get_permalink() here would return the trashed URL and md5() would miss the
7090		- // real vector IDs stored under the original URL.
7091		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
	3160	+ // Get the URL before post is deleted
	3161	+ $source_url = get_permalink($post_id);
7092	3162	if (!$source_url) {
7093		- //error_log('MXChat: Failed to resolve source URL for post ' . $post_id);
	3163	+ //error_log('MXChat: Failed to get permalink for post ' . $post_id);
7094	3164	return;
7095	3165	}
7096	3166
7097		- // Use chunk-aware deletion (handles both chunked and non-chunked content)
7098		- $delete_result = MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	3167	+ // Check if Pinecone is enabled
	3168	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	3169	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7099	3170
7100		- if (is_wp_error($delete_result)) {
7101		- //error_log('MXChat: Chunk-aware deletion failed for URL: ' . $source_url . ' - ' . $delete_result->get_error_message());
	3171	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3172	+ // Delete from Pinecone
	3173	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
	3174	+ } else {
	3175	+ // Delete from WordPress DB
	3176	+ global $wpdb;
	3177	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	3178	+
	3179	+ $result = $wpdb->delete(
	3180	+ $table_name,
	3181	+ array('source_url' => $source_url),
	3182	+ array('%s')
	3183	+ );
	3184	+
	3185	+ if ($result === false) {
	3186	+ //error_log('MXChat: WordPress DB deletion failed for URL: ' . $source_url);
	3187	+ }
7102	3188	}
	3189	+}
7103	3190
7104		- delete_transient('mxchat_prev_url_' . $post_id);
7105		- delete_transient('mxchat_prev_status_' . $post_id);
7106		-}
7107	3191
7108		-/**
7109		- * Resolve the source URL for a post being trashed/deleted.
7110		- *
7111		- * Why: wp_trash_post appends "__trashed" to the slug before the wp_trash_post action fires, so
7112		- * get_permalink() returns a URL whose md5() won't match the vector IDs stored in Pinecone or
7113		- * the source_url rows in the WP DB. Prefer the URL captured by mxchat_store_pre_update_status
7114		- * (runs on pre_post_update, before the rename); fall back to stripping the __trashed suffix.
7115		- */
7116		-private function mxchat_resolve_pre_trash_url($post_id) {
7117		- $previous_url = get_transient('mxchat_prev_url_' . $post_id);
7118		- if (!empty($previous_url)) {
7119		- return $previous_url;
7120		- }
	3192	+ /**
	3193	+ * Deletes data from Pinecone using a source URL
	3194	+ */
	3195	+ public function mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options) {
	3196	+ $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
	3197	+ $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
7121	3198
7122		- $current = get_permalink($post_id);
7123		- if (!$current) {
7124		- return '';
7125		- }
7126		- return preg_replace('#__trashed(/?)$#', '$1', $current);
7127		-}
	3199	+ if (empty($host) || empty($api_key)) {
	3200	+ //error_log('MXChat: Pinecone deletion failed - missing configuration');
	3201	+ return false;
	3202	+ }
7128	3203
	3204	+ $api_endpoint = "https://{$host}/vectors/delete";
	3205	+ $vector_id = md5($source_url);
7129	3206
	3207	+ $request_body = array(
	3208	+ 'ids' => array($vector_id)
	3209	+ );
7130	3210
	3211	+ $response = wp_remote_post($api_endpoint, array(
	3212	+ 'headers' => array(
	3213	+ 'Api-Key' => $api_key,
	3214	+ 'accept' => 'application/json',
	3215	+ 'content-type' => 'application/json'
	3216	+ ),
	3217	+ 'body' => wp_json_encode($request_body),
	3218	+ 'timeout' => 30
	3219	+ ));
	3220	+
	3221	+ if (is_wp_error($response)) {
	3222	+ //error_log('MXChat: Pinecone deletion error - ' . $response->get_error_message());
	3223	+ return false;
	3224	+ }
	3225	+
	3226	+ $response_code = wp_remote_retrieve_response_code($response);
	3227	+ if ($response_code !== 200) {
	3228	+ //error_log('MXChat: Pinecone deletion failed with status ' . $response_code);
	3229	+ return false;
	3230	+ }
	3231	+
	3232	+ return true;
	3233	+ }
	3234	+
	3235	+
	3236	+
7131	3237	public function mxchat_handle_product_change($post_id, $post, $update) {
7132	3238	if ($post->post_type !== 'product') {
7133	3239	return;
7134	3240	}
		@@ -7150,25 +3256,57 @@
7150	3256	if (!isset($this->options['enable_woocommerce_integration']) ||
7151	3257	!in_array($this->options['enable_woocommerce_integration'], ['1', 'on'])) {
7152	3258	return;
7153	3259	}
7154		-
	3260	+
7155	3261	$source_url = get_permalink($product->get_id());
7156		- $product_id = $product->get_id();
7157		-
7158		- // Build product content via the shared WC-object assembler (a3d60c) — this
7159		- // writer owns product rows whenever the integration is on.
7160		- $content = $this->mxchat_prepare_product_content_for_indexing($product);
7161		-
7162		- // Embedding decision — custom-provider-aware (plan cbd5fd); silent-return
7163		- // shape preserved.
7164		- $preflight = MxChat_Utils::embedding_preflight(get_option('mxchat_options'));
7165		- if (!$preflight['ok']) {
7166		- //error_log('MxChat Auto-sync: embedding pre-flight failed: ' . $preflight['reason']);
	3262	+
	3263	+ // Build product content
	3264	+ $title = $product->get_name();
	3265	+ $description = $product->get_description();
	3266	+ $short_description = $product->get_short_description();
	3267	+ $regular_price = $product->get_regular_price();
	3268	+ $sale_price = $product->get_sale_price();
	3269	+ $sku = $product->get_sku();
	3270	+
	3271	+ // Format content consistently
	3272	+ $content = $title . "\n\n";
	3273	+
	3274	+ if (!empty($description)) {
	3275	+ $content .= wp_strip_all_tags($description) . "\n\n";
	3276	+ }
	3277	+
	3278	+ if (!empty($short_description)) {
	3279	+ $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
	3280	+ }
	3281	+
	3282	+ $content .= "Price: $" . $regular_price . "\n";
	3283	+
	3284	+ if (!empty($sale_price)) {
	3285	+ $content .= "Sale Price: $" . $sale_price . "\n";
	3286	+ }
	3287	+
	3288	+ if (!empty($sku)) {
	3289	+ $content .= "SKU: " . $sku . "\n";
	3290	+ }
	3291	+
	3292	+ // Get API key with proper model detection
	3293	+ $options = get_option('mxchat_options');
	3294	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	3295	+
	3296	+ if (strpos($selected_model, 'voyage') === 0) {
	3297	+ $api_key = $options['voyage_api_key'] ?? '';
	3298	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	3299	+ $api_key = $options['gemini_api_key'] ?? '';
	3300	+ } else {
	3301	+ $api_key = $options['api_key'] ?? '';
	3302	+ }
	3303	+
	3304	+ if (empty($api_key)) {
	3305	+ //error_log('MxChat Auto-sync: No API key configured for embedding model');
7167	3306	return;
7168	3307	}
7169		- $api_key = $preflight['api_key'];
7170		-
	3308	+
7171	3309	// Use the centralized utility function for storage
7172	3310	$result = MxChat_Utils::submit_content_to_db(
7173	3311	$content,
7174	3312	$source_url,
		@@ -7174,16 +3312,11 @@
7174	3312	$source_url,
7175	3313	$api_key,
7176	3314	md5($source_url) // Vector ID for Pinecone
7177	3315	);
7178		-
7179		- // After successful storage, apply role restriction based on tags
7180		- if (!is_wp_error($result)) {
7181		- $this->apply_role_restriction_to_post($product_id, $source_url);
7182		- }
7183		-
	3316	+
7184	3317	if (is_wp_error($result)) {
7185		- //error_log('MxChat WooCommerce sync failed for product ' . $product_id . ': ' . $result->get_error_message());
	3318	+ //error_log('MxChat WooCommerce sync failed for product ' . $product->get_id() . ': ' . $result->get_error_message());
7186	3319	}
7187	3320	}
7188	3321
7189	3322	public function mxchat_handle_product_delete($post_id) {
		@@ -7190,18 +3323,28 @@
7190	3323	if (get_post_type($post_id) !== 'product') {
7191	3324	return;
7192	3325	}
7193	3326
7194		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
7195		- if (!$source_url) {
7196		- return;
7197		- }
	3327	+ $source_url = get_permalink($post_id);
7198	3328
7199		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
7200		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	3329	+ // Check if Pinecone is enabled
	3330	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	3331	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7201	3332
7202		- delete_transient('mxchat_prev_url_' . $post_id);
7203		- delete_transient('mxchat_prev_status_' . $post_id);
	3333	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3334	+ // Delete from Pinecone
	3335	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
	3336	+ } else {
	3337	+ // Delete from WordPress DB
	3338	+ global $wpdb;
	3339	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	3340	+
	3341	+ $wpdb->delete(
	3342	+ $table_name,
	3343	+ array('source_url' => $source_url),
	3344	+ array('%s')
	3345	+ );
	3346	+ }
7204	3347	}
7205	3348
7206	3349	/**
7207	3350	* Handle individual Pinecone content deletion
		@@ -7243,23 +3386,29 @@
7243	3386
7244	3387	// Delete from Pinecone
7245	3388	$pinecone_manager = MxChat_Pinecone_Manager::get_instance();
7246	3389	$result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
7247		- $vector_id,
7248		- $pinecone_options['mxchat_pinecone_api_key'],
7249		- $pinecone_options['mxchat_pinecone_host'],
7250		- $pinecone_options['mxchat_pinecone_namespace'] ?? ''
	3390	+ $vector_id,
	3391	+ $pinecone_options['mxchat_pinecone_api_key'],
	3392	+ $pinecone_options['mxchat_pinecone_host']
7251	3393	);
7252	3394
7253	3395	if ($result['success']) {
7254		- // Mirror the removal to the OpenAI Vector Store mapping (plan 15b5c6);
7255		- // a chunk vector id reduces to its base entry there.
7256		- if (class_exists('MxChat_Vectorstore_Manager')) {
7257		- MxChat_Vectorstore_Manager::sync_delete_by_key($vector_id, 'default');
7258		- }
7259		- // No cache clearing needed since we removed caching
7260		- set_transient('mxchat_admin_notice_success',
7261		- esc_html__('Entry deleted successfully from Pinecone.', 'mxchat'),
	3396	+ // Remove from ALL caches
	3397	+ $pinecone_manager->mxchat_remove_from_pinecone_vector_cache($vector_id);
	3398	+ $pinecone_manager->mxchat_remove_from_processed_content_caches($vector_id);
	3399	+
	3400	+ // CLEAR ALL RELEVANT CACHES
	3401	+ delete_transient('mxchat_pinecone_recent_1k_cache');
	3402	+ delete_option('mxchat_pinecone_vector_ids_cache');
	3403	+ delete_option('mxchat_pinecone_processed_cache');
	3404	+ delete_option('mxchat_processed_content_cache');
	3405	+
	3406	+ // Also force refresh for next page load
	3407	+ $pinecone_manager->mxchat_refresh_after_new_content($pinecone_options);
	3408	+
	3409	+ set_transient('mxchat_admin_notice_success',
	3410	+ esc_html__('Entry deleted successfully from Pinecone.', 'mxchat'),
7262	3411	30
7263	3412	);
7264	3413	} else {
7265	3414	set_transient('mxchat_admin_notice_error',
		@@ -7270,11 +3419,9 @@
7270	3419
7271	3420	wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
7272	3421	exit;
7273	3422	}
7274		-/**
7275		- * Handle individual Pinecone content deletion via AJAX
7276		- */
	3423	+
7277	3424	public function ajax_mxchat_delete_pinecone_prompt() {
7278	3425	// Verify nonce and permissions
7279	3426	if (!check_ajax_referer('mxchat_delete_pinecone_prompt_nonce', 'nonce', false)) {
7280	3427	wp_send_json_error('Invalid nonce');
		@@ -7286,9 +3433,8 @@
7286	3433	exit;
7287	3434	}
7288	3435
7289	3436	$vector_id = isset($_POST['vector_id']) ? sanitize_text_field($_POST['vector_id']) : '';
7290		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
7291	3437
7292	3438	if (empty($vector_id)) {
7293	3439	wp_send_json_error('Missing vector ID');
7294	3440	exit;
		@@ -7293,1147 +3439,50 @@
7293	3439	wp_send_json_error('Missing vector ID');
7294	3440	exit;
7295	3441	}
7296	3442
7297		- // Get bot-specific Pinecone settings
7298		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
7299		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
7300		-
	3443	+ // Get Pinecone settings
	3444	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
7301	3445	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7302	3446
7303	3447	if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
7304		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
	3448	+ wp_send_json_error('Pinecone is not properly configured');
7305	3449	exit;
7306	3450	}
7307	3451
7308		- // Delete from the correct Pinecone index and namespace
	3452	+ // Delete from Pinecone
	3453	+ $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
7309	3454	$result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
7310		- $vector_id,
7311		- $pinecone_options['mxchat_pinecone_api_key'],
7312		- $pinecone_options['mxchat_pinecone_host'],
7313		- $pinecone_options['mxchat_pinecone_namespace'] ?? ''
	3455	+ $vector_id,
	3456	+ $pinecone_options['mxchat_pinecone_api_key'],
	3457	+ $pinecone_options['mxchat_pinecone_host']
7314	3458	);
7315	3459
7316	3460	if ($result['success']) {
7317		- // Mirror the removal to the OpenAI Vector Store mapping (plan 15b5c6)
7318		- if (class_exists('MxChat_Vectorstore_Manager')) {
7319		- MxChat_Vectorstore_Manager::sync_delete_by_key($vector_id, $bot_id);
7320		- }
7321		- // No cache clearing needed since we removed caching
	3461	+ // Remove from ALL caches
	3462	+ $pinecone_manager->mxchat_remove_from_pinecone_vector_cache($vector_id);
	3463	+ $pinecone_manager->mxchat_remove_from_processed_content_caches($vector_id);
	3464	+
	3465	+ // CLEAR ALL RELEVANT CACHES (ADD THESE LINES)
	3466	+ delete_transient('mxchat_pinecone_recent_1k_cache');
	3467	+ delete_option('mxchat_pinecone_vector_ids_cache');
	3468	+ delete_option('mxchat_pinecone_processed_cache');
	3469	+ delete_option('mxchat_processed_content_cache');
	3470	+
	3471	+ // Also force refresh for next page load
	3472	+ $pinecone_manager->mxchat_refresh_after_new_content($pinecone_options);
	3473	+
7322	3474	wp_send_json_success(array(
7323	3475	'message' => 'Entry deleted successfully from Pinecone',
7324		- 'vector_id' => $vector_id,
7325		- 'bot_id' => $bot_id
	3476	+ 'vector_id' => $vector_id
7326	3477	));
7327	3478	} else {
7328		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete from Pinecone: ' . $result['message'], array('vector_id' => $vector_id, 'bot_id' => $bot_id));
7329	3479	wp_send_json_error('Failed to delete from Pinecone: ' . $result['message']);
7330	3480	}
7331	3481
7332	3482	exit;
7333	3483	}
7334		-
7335		-/**
7336		- * Handle deletion of all chunks for a given source URL via AJAX
7337		- * Follows the same pattern as ajax_mxchat_delete_pinecone_prompt
7338		- */
7339		-public function ajax_mxchat_delete_chunks_by_url() {
7340		- // Verify nonce and permissions
7341		- if (!check_ajax_referer('mxchat_delete_chunks_nonce', 'nonce', false)) {
7342		- wp_send_json_error('Invalid nonce');
7343		- exit;
7344		- }
7345		-
7346		- if (!current_user_can('manage_options')) {
7347		- wp_send_json_error('Unauthorized access');
7348		- exit;
7349		- }
7350		-
7351		- $source_url = isset($_POST['source_url']) ? esc_url_raw($_POST['source_url']) : '';
7352		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
7353		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
7354		-
7355		- if (empty($source_url)) {
7356		- wp_send_json_error('Missing source URL');
7357		- exit;
7358		- }
7359		-
7360		- // Generate the base vector ID from the source URL (same as how chunks are created)
7361		- $base_vector_id = md5($source_url);
7362		-
7363		- if ($data_source === 'pinecone') {
7364		- // Get bot-specific Pinecone settings (same as working delete function)
7365		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
7366		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
7367		-
7368		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7369		-
7370		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
7371		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
7372		- exit;
7373		- }
7374		-
7375		- $api_key = $pinecone_options['mxchat_pinecone_api_key'];
7376		- $host = $pinecone_options['mxchat_pinecone_host'];
7377		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
7378		-
7379		- // Collect all vector IDs to delete
7380		- $vectors_to_delete = array();
7381		-
7382		- // Add the original single-vector ID (for non-chunked content)
7383		- $vectors_to_delete[] = $base_vector_id;
7384		-
7385		- // Use Pinecone list API to find all chunk vectors with this prefix
7386		- // NOTE: Pinecone List API is a GET request with query parameters, not POST
7387		- $prefix = $base_vector_id . '_chunk_';
7388		-
7389		- $query_params = array(
7390		- 'prefix' => $prefix,
7391		- 'limit' => 100
7392		- );
7393		-
7394		- if (!empty($namespace)) {
7395		- $query_params['namespace'] = $namespace;
7396		- }
7397		-
7398		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
7399		-
7400		- $list_response = wp_remote_get($list_url, array(
7401		- 'headers' => array(
7402		- 'Api-Key' => $api_key,
7403		- 'accept' => 'application/json'
7404		- ),
7405		- 'timeout' => 30
7406		- ));
7407		-
7408		- if (!is_wp_error($list_response)) {
7409		- $list_body_response = wp_remote_retrieve_body($list_response);
7410		- $list_data = json_decode($list_body_response, true);
7411		- if (!empty($list_data['vectors'])) {
7412		- foreach ($list_data['vectors'] as $vector) {
7413		- if (isset($vector['id'])) {
7414		- $vectors_to_delete[] = $vector['id'];
7415		- }
7416		- }
7417		- }
7418		- }
7419		-
7420		- if (empty($vectors_to_delete)) {
7421		- // Entry already gone from Pinecone — still clear any mirrored
7422		- // Vector Store file so it can't outlive the entry (plan 15b5c6).
7423		- if (class_exists('MxChat_Vectorstore_Manager')) {
7424		- MxChat_Vectorstore_Manager::sync_delete_entry($source_url, $bot_id);
7425		- }
7426		- wp_send_json_success(array(
7427		- 'message' => 'No vectors found to delete',
7428		- 'source_url' => $source_url
7429		- ));
7430		- exit;
7431		- }
7432		-
7433		- // Delete all vectors using the same endpoint as the working function
7434		- $delete_url = "https://{$host}/vectors/delete";
7435		-
7436		- $delete_body = array(
7437		- 'ids' => $vectors_to_delete
7438		- );
7439		-
7440		- if (!empty($namespace)) {
7441		- $delete_body['namespace'] = $namespace;
7442		- }
7443		-
7444		- $delete_response = wp_remote_post($delete_url, array(
7445		- 'headers' => array(
7446		- 'Api-Key' => $api_key,
7447		- 'accept' => 'application/json',
7448		- 'content-type' => 'application/json'
7449		- ),
7450		- 'body' => wp_json_encode($delete_body),
7451		- 'timeout' => 30
7452		- ));
7453		-
7454		- if (is_wp_error($delete_response)) {
7455		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete chunks from Pinecone: ' . $delete_response->get_error_message(), array('source_url' => $source_url));
7456		- wp_send_json_error('Failed to delete from Pinecone: ' . $delete_response->get_error_message());
7457		- exit;
7458		- }
7459		-
7460		- $response_code = wp_remote_retrieve_response_code($delete_response);
7461		-
7462		- if ($response_code !== 200) {
7463		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Pinecone API error (HTTP ' . $response_code . ')', array('source_url' => $source_url));
7464		- wp_send_json_error('Pinecone API error (HTTP ' . $response_code . ')');
7465		- exit;
7466		- }
7467		-
7468		- // Mirror the removal to the OpenAI Vector Store (plan 15b5c6)
7469		- if (class_exists('MxChat_Vectorstore_Manager')) {
7470		- MxChat_Vectorstore_Manager::sync_delete_entry($source_url, $bot_id);
7471		- }
7472		-
7473		- wp_send_json_success(array(
7474		- 'message' => 'All chunks deleted successfully from Pinecone',
7475		- 'source_url' => $source_url,
7476		- 'deleted_count' => count($vectors_to_delete)
7477		- ));
7478		-
7479		- } else {
7480		- // WordPress database deletion
7481		- global $wpdb;
7482		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7483		-
7484		- $result = $wpdb->delete(
7485		- $table_name,
7486		- array('source_url' => $source_url),
7487		- array('%s')
7488		- );
7489		-
7490		- if ($result === false) {
7491		- MxChat_Admin::mxchat_log_debug('database_error', 'Failed to delete from database: ' . $wpdb->last_error, array('source_url' => $source_url));
7492		- wp_send_json_error('Failed to delete from database: ' . $wpdb->last_error);
7493		- exit;
7494		- }
7495		-
7496		- // Mirror the removal to the OpenAI Vector Store (plan 15b5c6)
7497		- if (class_exists('MxChat_Vectorstore_Manager')) {
7498		- MxChat_Vectorstore_Manager::sync_delete_entry($source_url, $bot_id);
7499		- }
7500		-
7501		- wp_send_json_success(array(
7502		- 'message' => 'All chunks deleted successfully from database',
7503		- 'source_url' => $source_url,
7504		- 'deleted_count' => $result
7505		- ));
7506		- }
7507		-
7508		- exit;
7509		-}
7510		-
7511		-/**
7512		- * Handle individual WordPress database content deletion via AJAX
7513		- * Mirrors the Pinecone delete handler but for WordPress database entries
7514		- */
7515		-public function ajax_mxchat_delete_wordpress_prompt() {
7516		- // Verify nonce and permissions
7517		- if (!check_ajax_referer('mxchat_delete_wordpress_prompt_nonce', 'nonce', false)) {
7518		- wp_send_json_error('Invalid nonce');
7519		- exit;
7520		- }
7521		-
7522		- if (!current_user_can('manage_options')) {
7523		- wp_send_json_error('Unauthorized access');
7524		- exit;
7525		- }
7526		-
7527		- $entry_id = isset($_POST['entry_id']) ? intval($_POST['entry_id']) : 0;
7528		-
7529		- if (empty($entry_id)) {
7530		- wp_send_json_error('Missing entry ID');
7531		- exit;
7532		- }
7533		-
7534		- global $wpdb;
7535		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7536		-
7537		- // Capture the identity BEFORE the row disappears — needed to mirror the
7538		- // change into the Vector Store (plan 15b5c6).
7539		- $source_url = $wpdb->get_var($wpdb->prepare(
7540		- "SELECT source_url FROM {$table_name} WHERE id = %d",
7541		- $entry_id
7542		- ));
7543		-
7544		- // Clear cache for this entry
7545		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
7546		-
7547		- // Delete from database
7548		- $result = $wpdb->delete(
7549		- $table_name,
7550		- array('id' => $entry_id),
7551		- array('%d')
7552		- );
7553		-
7554		- if ($result !== false) {
7555		- // Mirror to the Vector Store: if sibling rows remain (this was one
7556		- // chunk of a larger entry) the entry's file is REFRESHED from what's
7557		- // left; if none remain, the file is removed.
7558		- if (!empty($source_url) && class_exists('MxChat_Vectorstore_Manager')) {
7559		- $remaining = (int) $wpdb->get_var($wpdb->prepare(
7560		- "SELECT COUNT(*) FROM {$table_name} WHERE source_url = %s",
7561		- $source_url
7562		- ));
7563		- if ($remaining > 0) {
7564		- MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, '', 'default');
7565		- } else {
7566		- MxChat_Vectorstore_Manager::sync_delete_entry($source_url, 'default');
7567		- }
7568		- }
7569		-
7570		- wp_send_json_success(array(
7571		- 'message' => 'Entry deleted successfully',
7572		- 'entry_id' => $entry_id
7573		- ));
7574		- } else {
7575		- wp_send_json_error('Failed to delete entry from database');
7576		- }
7577		-
7578		- exit;
7579		-}
7580		-
7581		-/**
7582		- * Handle bulk deletion of knowledge entries via AJAX
7583		- * Supports both Pinecone and WordPress database entries
7584		- */
7585		-public function ajax_mxchat_bulk_delete_knowledge() {
7586		- // Verify nonce and permissions
7587		- if (!check_ajax_referer('mxchat_bulk_delete_knowledge_nonce', 'nonce', false)) {
7588		- wp_send_json_error('Invalid nonce');
7589		- exit;
7590		- }
7591		-
7592		- if (!current_user_can('manage_options')) {
7593		- wp_send_json_error('Unauthorized access');
7594		- exit;
7595		- }
7596		-
7597		- $entries = isset($_POST['entries']) ? $_POST['entries'] : array();
7598		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
7599		-
7600		- if (empty($entries) || !is_array($entries)) {
7601		- wp_send_json_error('No entries provided');
7602		- exit;
7603		- }
7604		-
7605		- // Extend execution time — bulk Pinecone operations can take a while
7606		- if (function_exists('set_time_limit')) {
7607		- set_time_limit(120);
7608		- }
7609		-
7610		- $success_ids = array();
7611		- $failed_ids = array();
7612		- $errors = array();
7613		-
7614		- global $wpdb;
7615		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7616		-
7617		- // Get Pinecone manager for Pinecone deletions
7618		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
7619		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
7620		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7621		-
7622		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
7623		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
7624		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
7625		-
7626		- // =============================================
7627		- // PHASE 1: Collect all Pinecone vector IDs
7628		- // and separate WordPress entries
7629		- // =============================================
7630		- $pinecone_entry_ids = array(); // entry IDs that are pinecone-sourced
7631		- $wordpress_entries = array(); // entries for WordPress DB deletion
7632		- $all_vector_ids = array(); // all pinecone vector IDs to delete in one batch
7633		- $vs_mirror_urls = array(); // Vector Store mirror: URLs to delete (plan 15b5c6)
7634		- $vs_mirror_keys = array(); // Vector Store mirror: bare vector ids to delete
7635		-
7636		- foreach ($entries as $entry) {
7637		- $entry_id = sanitize_text_field($entry['id'] ?? '');
7638		- $source = sanitize_text_field($entry['source'] ?? 'wordpress');
7639		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
7640		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
7641		-
7642		- if (empty($entry_id)) {
7643		- continue;
7644		- }
7645		-
7646		- if ($source === 'pinecone') {
7647		- if (!$use_pinecone || empty($api_key)) {
7648		- $failed_ids[] = $entry_id;
7649		- $errors[] = "Pinecone not configured for entry: $entry_id";
7650		- continue;
7651		- }
7652		-
7653		- $pinecone_entry_ids[] = $entry_id;
7654		-
7655		- if ($is_group && !empty($source_url)) {
7656		- // Grouped/chunked entry: collect base ID + chunk IDs via List API
7657		- $base_vector_id = md5($source_url);
7658		- $all_vector_ids[] = $base_vector_id;
7659		-
7660		- $list_url = 'https://' . $host . '/vectors/list?prefix=' . urlencode($base_vector_id . '_chunk_') . '&limit=100';
7661		- if (!empty($namespace)) {
7662		- $list_url .= '&namespace=' . rawurlencode($namespace);
7663		- }
7664		- $list_response = wp_remote_get($list_url, array(
7665		- 'headers' => array(
7666		- 'Api-Key' => $api_key,
7667		- 'accept' => 'application/json'
7668		- ),
7669		- 'timeout' => 30
7670		- ));
7671		-
7672		- if (!is_wp_error($list_response)) {
7673		- $list_body = json_decode(wp_remote_retrieve_body($list_response), true);
7674		- if (!empty($list_body['vectors']) && is_array($list_body['vectors'])) {
7675		- foreach ($list_body['vectors'] as $vector) {
7676		- if (isset($vector['id'])) {
7677		- $all_vector_ids[] = $vector['id'];
7678		- }
7679		- }
7680		- }
7681		- }
7682		- } else {
7683		- // Single entry: the entry_id IS the vector ID
7684		- $all_vector_ids[] = $entry_id;
7685		- }
7686		-
7687		- if (!empty($source_url)) {
7688		- $vs_mirror_urls[] = $source_url;
7689		- } else {
7690		- $vs_mirror_keys[] = $entry_id;
7691		- }
7692		- } else {
7693		- $wordpress_entries[] = $entry;
7694		- }
7695		- }
7696		-
7697		- // =============================================
7698		- // PHASE 2: Single batch delete to Pinecone
7699		- // =============================================
7700		- if (!empty($all_vector_ids)) {
7701		- $all_vector_ids = array_values(array_unique($all_vector_ids));
7702		- $pinecone_success = true;
7703		- $batches = array_chunk($all_vector_ids, 100);
7704		-
7705		- foreach ($batches as $batch) {
7706		- $delete_body = array('ids' => $batch);
7707		- if (!empty($namespace)) {
7708		- $delete_body['namespace'] = $namespace;
7709		- }
7710		- $delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
7711		- 'headers' => array(
7712		- 'Api-Key' => $api_key,
7713		- 'accept' => 'application/json',
7714		- 'content-type' => 'application/json'
7715		- ),
7716		- 'body' => wp_json_encode($delete_body),
7717		- 'timeout' => 60
7718		- ));
7719		-
7720		- if (is_wp_error($delete_response)) {
7721		- $pinecone_success = false;
7722		- $errors[] = 'Pinecone batch deletion failed: ' . $delete_response->get_error_message();
7723		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed: ' . $delete_response->get_error_message());
7724		- } else {
7725		- $response_code = wp_remote_retrieve_response_code($delete_response);
7726		- if ($response_code !== 200) {
7727		- $pinecone_success = false;
7728		- $response_body = wp_remote_retrieve_body($delete_response);
7729		- $errors[] = "Pinecone API error (HTTP $response_code)";
7730		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed (HTTP ' . $response_code . ')', array('response' => substr($response_body, 0, 200)));
7731		- }
7732		- }
7733		- }
7734		-
7735		- // Mark all pinecone entries based on batch result
7736		- foreach ($pinecone_entry_ids as $eid) {
7737		- if ($pinecone_success) {
7738		- $success_ids[] = $eid;
7739		- } else {
7740		- $failed_ids[] = $eid;
7741		- }
7742		- }
7743		-
7744		- // Mirror the removals to the OpenAI Vector Store (plan 15b5c6)
7745		- if ($pinecone_success && class_exists('MxChat_Vectorstore_Manager')) {
7746		- foreach (array_unique($vs_mirror_urls) as $vs_url) {
7747		- MxChat_Vectorstore_Manager::sync_delete_entry($vs_url, $bot_id);
7748		- }
7749		- foreach (array_unique($vs_mirror_keys) as $vs_key) {
7750		- MxChat_Vectorstore_Manager::sync_delete_by_key($vs_key, $bot_id);
7751		- }
7752		- }
7753		- }
7754		-
7755		- // =============================================
7756		- // PHASE 3: WordPress database deletions
7757		- // =============================================
7758		- foreach ($wordpress_entries as $entry) {
7759		- $entry_id = sanitize_text_field($entry['id'] ?? '');
7760		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
7761		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
7762		-
7763		- if (empty($entry_id)) {
7764		- continue;
7765		- }
7766		-
7767		- try {
7768		- if ($is_group && !empty($source_url)) {
7769		- $result = $wpdb->delete(
7770		- $table_name,
7771		- array('source_url' => $source_url),
7772		- array('%s')
7773		- );
7774		- $row_url = $source_url;
7775		- } else {
7776		- // Identity captured pre-delete for the Vector Store mirror
7777		- $row_url = $wpdb->get_var($wpdb->prepare(
7778		- "SELECT source_url FROM {$table_name} WHERE id = %d",
7779		- intval($entry_id)
7780		- ));
7781		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
7782		- $result = $wpdb->delete(
7783		- $table_name,
7784		- array('id' => intval($entry_id)),
7785		- array('%d')
7786		- );
7787		- }
7788		-
7789		- if ($result !== false) {
7790		- $success_ids[] = $entry_id;
7791		- // Mirror to the Vector Store: refresh the entry's file when
7792		- // sibling chunk rows survive, remove it when none do.
7793		- if (!empty($row_url) && class_exists('MxChat_Vectorstore_Manager')) {
7794		- $remaining = (int) $wpdb->get_var($wpdb->prepare(
7795		- "SELECT COUNT(*) FROM {$table_name} WHERE source_url = %s",
7796		- $row_url
7797		- ));
7798		- if ($remaining > 0) {
7799		- MxChat_Vectorstore_Manager::sync_upsert_entry($row_url, '', $bot_id);
7800		- } else {
7801		- MxChat_Vectorstore_Manager::sync_delete_entry($row_url, $bot_id);
7802		- }
7803		- }
7804		- } else {
7805		- $failed_ids[] = $entry_id;
7806		- $errors[] = "Database error for entry: $entry_id";
7807		- }
7808		- } catch (Exception $e) {
7809		- $failed_ids[] = $entry_id;
7810		- $errors[] = $e->getMessage();
7811		- }
7812		- }
7813		-
7814		- wp_send_json_success(array(
7815		- 'success_ids' => $success_ids,
7816		- 'failed_ids' => $failed_ids,
7817		- 'errors' => $errors,
7818		- 'total_processed' => count($success_ids) + count($failed_ids)
7819		- ));
7820		-
7821		- exit;
7822		-}
7823		-
7824		-/**
7825		- * Get hierarchical roles for dropdown
7826		- */
7827		-public function mxchat_get_role_options() {
7828		- return array(
7829		- 'public' => __('Public (Everyone)', 'mxchat'),
7830		- 'logged_in' => __('Logged In Users', 'mxchat'),
7831		- 'subscriber' => __('Subscribers & Above', 'mxchat'),
7832		- 'contributor' => __('Contributors & Above', 'mxchat'),
7833		- 'author' => __('Authors & Above', 'mxchat'),
7834		- 'editor' => __('Editors & Above', 'mxchat'),
7835		- 'administrator' => __('Administrators Only', 'mxchat')
7836		- );
7837		-}
7838		-
7839		-/**
7840		- * Check if user has access to content based on role restriction
7841		- */
7842		-public function mxchat_user_has_content_access($role_restriction) {
7843		- // Public content is always accessible
7844		- if ($role_restriction === 'public' || empty($role_restriction)) {
7845		- return true;
7846		- }
7847	3484
7848		- // Check if user is logged in for logged_in restriction
7849		- if ($role_restriction === 'logged_in') {
7850		- return is_user_logged_in();
7851		- }
7852		-
7853		- // If not logged in, no access to role-restricted content
7854		- if (!is_user_logged_in()) {
7855		- return false;
7856		- }
7857		-
7858		- $user = wp_get_current_user();
7859		- $user_roles = $user->roles;
7860		-
7861		- if (empty($user_roles)) {
7862		- return false;
7863		- }
7864		-
7865		- // Define role hierarchy (higher number = higher access)
7866		- $hierarchy = array(
7867		- 'subscriber' => 1,
7868		- 'contributor' => 2,
7869		- 'author' => 3,
7870		- 'editor' => 4,
7871		- 'administrator' => 5
7872		- );
7873		-
7874		- // Get required level
7875		- $required_level = isset($hierarchy[$role_restriction]) ? $hierarchy[$role_restriction] : 0;
7876		-
7877		- // Check if user has required level or higher
7878		- foreach ($user_roles as $user_role) {
7879		- $user_level = isset($hierarchy[$user_role]) ? $hierarchy[$user_role] : 0;
7880		- if ($user_level >= $required_level) {
7881		- return true;
7882		- }
7883		- }
7884		-
7885		- return false;
7886		-}
7887		-
7888		-/**
7889		- * Handle role restriction updates via AJAX
7890		- * Removed cache clearing call since we removed caching
7891		- */
7892		-public function ajax_mxchat_update_role_restriction() {
7893		- // Verify nonce and permissions
7894		- if (!check_ajax_referer('mxchat_update_role_nonce', 'nonce', false)) {
7895		- wp_send_json_error('Invalid nonce');
7896		- exit;
7897		- }
7898		-
7899		- if (!current_user_can('manage_options')) {
7900		- wp_send_json_error('Unauthorized access');
7901		- exit;
7902		- }
7903		-
7904		- $entry_id = isset($_POST['entry_id']) ? sanitize_text_field($_POST['entry_id']) : '';
7905		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
7906		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
7907		-
7908		- if (empty($entry_id)) {
7909		- wp_send_json_error('Invalid entry ID');
7910		- exit;
7911		- }
7912		-
7913		- // Get knowledge manager instance to validate role restriction
7914		- $knowledge_manager = MxChat_Knowledge_Manager::get_instance();
7915		- $valid_roles = array_keys($knowledge_manager->mxchat_get_role_options());
7916		- if (!in_array($role_restriction, $valid_roles)) {
7917		- wp_send_json_error('Invalid role restriction');
7918		- exit;
7919		- }
7920		-
7921		- global $wpdb;
7922		-
7923		- if ($data_source === 'pinecone') {
7924		- // Handle Pinecone role restriction (stored separately in WordPress table)
7925		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7926		-
7927		- // Use REPLACE to insert or update the role restriction
7928		- $result = $wpdb->replace(
7929		- $roles_table,
7930		- array(
7931		- 'vector_id' => $entry_id,
7932		- 'role_restriction' => $role_restriction,
7933		- 'updated_at' => current_time('mysql')
7934		- ),
7935		- array('%s', '%s', '%s')
7936		- );
7937		-
7938		- // No cache clearing needed since we removed caching
7939		-
7940		- } else {
7941		- // Handle WordPress database role restriction (existing functionality)
7942		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7943		-
7944		- $result = $wpdb->update(
7945		- $table_name,
7946		- array('role_restriction' => $role_restriction),
7947		- array('id' => absint($entry_id)),
7948		- array('%s'),
7949		- array('%d')
7950		- );
7951		- }
7952		-
7953		- if ($result === false) {
7954		- wp_send_json_error('Database update failed: ' . $wpdb->last_error);
7955		- exit;
7956		- }
7957		-
7958		- // Keep the OpenAI Vector Store mirror consistent with the new restriction
7959		- // (plan 15b5c6): non-public pulls the mirrored file, public re-mirrors.
7960		- if (class_exists('MxChat_Vectorstore_Manager')) {
7961		- if ($data_source === 'pinecone') {
7962		- if ($role_restriction !== 'public') {
7963		- MxChat_Vectorstore_Manager::sync_delete_by_key($entry_id, 'default');
7964		- }
7965		- // Public again: Pinecone-mode content isn't held locally, so the
7966		- // entry re-mirrors on its next save/import rather than here.
7967		- } else {
7968		- $row_url = $wpdb->get_var($wpdb->prepare(
7969		- "SELECT source_url FROM {$wpdb->prefix}mxchat_system_prompt_content WHERE id = %d",
7970		- absint($entry_id)
7971		- ));
7972		- if (!empty($row_url)) {
7973		- MxChat_Vectorstore_Manager::handle_role_change($row_url, 'default', $role_restriction);
7974		- }
7975		- }
7976		- }
7977		-
7978		- wp_send_json_success(array(
7979		- 'message' => 'Role restriction updated successfully',
7980		- 'role_restriction' => $role_restriction,
7981		- 'data_source' => $data_source,
7982		- 'entry_id' => $entry_id
7983		- ));
7984		- exit;
7985		-}
7986		-
7987		-// ========================================
7988		-// ROLE-BASED CONTENT RESTRICTIONS - NEW FUNCTIONS
7989		-// Add these to your MxChat_Knowledge_Manager class
7990		-// ========================================
7991		-
7992		-/**
7993		- * Initialize role-based content hooks
7994		- * Add this call to your __construct() or mxchat_init_hooks() method
7995		- */
7996		-private function mxchat_init_role_hooks() {
7997		- // AJAX handlers for tag-role mappings
7998		- add_action('wp_ajax_mxchat_add_tag_role_mapping', array($this, 'ajax_add_tag_role_mapping'));
7999		- add_action('wp_ajax_mxchat_delete_tag_role_mapping', array($this, 'ajax_delete_tag_role_mapping'));
8000		- add_action('wp_ajax_mxchat_get_tag_role_mappings', array($this, 'ajax_get_tag_role_mappings'));
8001		- add_action('wp_ajax_mxchat_bulk_update_tag_roles', array($this, 'ajax_bulk_update_tag_roles'));
8002		-
8003		- // Hook to automatically update role restrictions when tags are added/removed
8004		- add_action('set_object_terms', array($this, 'handle_tag_change'), 10, 6);
8005		-
8006		- // Hook to apply role restrictions on auto-sync
8007		- add_action('mxchat_content_stored', array($this, 'apply_role_restriction_after_storage'), 10, 2);
8008		-}
8009		-
8010		-/**
8011		- * Add tag-role mapping via AJAX
8012		- */
8013		-public function ajax_add_tag_role_mapping() {
8014		- // Verify nonce and permissions
8015		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
8016		-
8017		- if (!current_user_can('manage_options')) {
8018		- wp_send_json_error('Unauthorized access');
8019		- exit;
8020		- }
8021		-
8022		- $tag_input = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
8023		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
8024		-
8025		- if (empty($tag_input)) {
8026		- wp_send_json_error('Please enter a tag name or slug');
8027		- exit;
8028		- }
8029		-
8030		- // Validate role restriction
8031		- $valid_roles = array_keys($this->mxchat_get_role_options());
8032		- if (!in_array($role_restriction, $valid_roles)) {
8033		- wp_send_json_error('Invalid role restriction');
8034		- exit;
8035		- }
8036		-
8037		- // Resolve the tag by slug first, then fall back to its display name, so users can
8038		- // enter either "premium-content" or "Premium Content". (plan b8bcf5 — the field is
8039		- // labeled by name but previously validated by slug only, producing the confusing
8040		- // "Tag does not exist in WordPress" error when a real tag's name was typed.)
8041		- $term = get_term_by('slug', $tag_input, 'post_tag');
8042		- if (!$term) {
8043		- $term = get_term_by('name', $tag_input, 'post_tag');
8044		- }
8045		- if (!$term) {
8046		- wp_send_json_error('No tag with that name or slug exists yet. Create it under Posts → Tags first, then enter its name or slug.');
8047		- exit;
8048		- }
8049		-
8050		- // Always key the mapping by the RESOLVED slug — apply_role_restriction_to_post()
8051		- // compares against each post's tag slugs, so the stored key must be a slug,
8052		- // never the raw (possibly display-name) input.
8053		- $tag_slug = $term->slug;
8054		-
8055		- // Get existing mappings
8056		- $mappings = get_option('mxchat_tag_role_mappings', array());
8057		-
8058		- // Check if mapping already exists
8059		- if (isset($mappings[$tag_slug])) {
8060		- wp_send_json_error('Mapping for this tag already exists');
8061		- exit;
8062		- }
8063		-
8064		- // Add new mapping
8065		- $mappings[$tag_slug] = $role_restriction;
8066		- update_option('mxchat_tag_role_mappings', $mappings);
8067		-
8068		- wp_send_json_success(array(
8069		- 'message' => 'Tag-role mapping added successfully',
8070		- 'tag_slug' => $tag_slug,
8071		- 'role_restriction' => $role_restriction
8072		- ));
8073		- exit;
8074		-}
8075		-
8076		-/**
8077		- * Delete tag-role mapping via AJAX
8078		- */
8079		-public function ajax_delete_tag_role_mapping() {
8080		- // Verify nonce and permissions
8081		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
8082		-
8083		- if (!current_user_can('manage_options')) {
8084		- wp_send_json_error('Unauthorized access');
8085		- exit;
8086		- }
8087		-
8088		- $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
8089		-
8090		- if (empty($tag_slug)) {
8091		- wp_send_json_error('Tag slug is required');
8092		- exit;
8093		- }
8094		-
8095		- // Get existing mappings
8096		- $mappings = get_option('mxchat_tag_role_mappings', array());
8097		-
8098		- // Check if mapping exists
8099		- if (!isset($mappings[$tag_slug])) {
8100		- wp_send_json_error('Mapping does not exist');
8101		- exit;
8102		- }
8103		-
8104		- // Remove mapping
8105		- unset($mappings[$tag_slug]);
8106		- update_option('mxchat_tag_role_mappings', $mappings);
8107		-
8108		- wp_send_json_success(array(
8109		- 'message' => 'Tag-role mapping deleted successfully',
8110		- 'tag_slug' => $tag_slug
8111		- ));
8112		- exit;
8113		-}
8114		-
8115		-/**
8116		- * Get all tag-role mappings via AJAX
8117		- */
8118		-public function ajax_get_tag_role_mappings() {
8119		- // Verify nonce and permissions
8120		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
8121		-
8122		- if (!current_user_can('manage_options')) {
8123		- wp_send_json_error('Unauthorized access');
8124		- exit;
8125		- }
8126		-
8127		- // Get mappings
8128		- $mappings = get_option('mxchat_tag_role_mappings', array());
8129		- $role_options = $this->mxchat_get_role_options();
8130		-
8131		- $formatted_mappings = array();
8132		-
8133		- foreach ($mappings as $tag_slug => $role_restriction) {
8134		- // Get tag object
8135		- $term = get_term_by('slug', $tag_slug, 'post_tag');
8136		-
8137		- // Count posts with this tag
8138		- $post_count = 0;
8139		- if ($term) {
8140		- $post_count = $term->count;
8141		- }
8142		-
8143		- $formatted_mappings[] = array(
8144		- 'tag_slug' => $tag_slug,
8145		- 'role_restriction' => $role_restriction,
8146		- 'role_label' => $role_options[$role_restriction] ?? $role_restriction,
8147		- 'post_count' => $post_count
8148		- );
8149		- }
8150		-
8151		- wp_send_json_success(array(
8152		- 'mappings' => $formatted_mappings
8153		- ));
8154		- exit;
8155		-}
8156		-
8157		-/**
8158		- * Bulk update role restrictions for all existing content with mapped tags
8159		- */
8160		-public function ajax_bulk_update_tag_roles() {
8161		- // Verify nonce and permissions
8162		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
8163		-
8164		- if (!current_user_can('manage_options')) {
8165		- wp_send_json_error('Unauthorized access');
8166		- exit;
8167		- }
8168		-
8169		- // Get mappings
8170		- $mappings = get_option('mxchat_tag_role_mappings', array());
8171		-
8172		- if (empty($mappings)) {
8173		- wp_send_json_error('No tag-role mappings found');
8174		- exit;
8175		- }
8176		-
8177		- global $wpdb;
8178		-
8179		- // Check if using Pinecone
8180		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
8181		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
8182		-
8183		- $updated_count = 0;
8184		- $details = array();
8185		-
8186		- foreach ($mappings as $tag_slug => $role_restriction) {
8187		- // Get all posts with this tag
8188		- $posts = get_posts(array(
8189		- 'tag' => $tag_slug,
8190		- 'post_type' => 'any',
8191		- 'posts_per_page' => -1,
8192		- 'fields' => 'ids',
8193		- 'post_status' => 'publish'
8194		- ));
8195		-
8196		- if (empty($posts)) {
8197		- continue;
8198		- }
8199		-
8200		- $tag_updated = 0;
8201		-
8202		- foreach ($posts as $post_id) {
8203		- $source_url = get_permalink($post_id);
8204		- if (!$source_url) {
8205		- continue;
8206		- }
8207		-
8208		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
8209		- // Update Pinecone role restriction
8210		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
8211		- $vector_id = md5($source_url);
8212		-
8213		- $result = $wpdb->replace(
8214		- $roles_table,
8215		- array(
8216		- 'vector_id' => $vector_id,
8217		- 'role_restriction' => $role_restriction,
8218		- 'updated_at' => current_time('mysql')
8219		- ),
8220		- array('%s', '%s', '%s')
8221		- );
8222		- } else {
8223		- // Update WordPress DB
8224		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
8225		-
8226		- $result = $wpdb->update(
8227		- $table_name,
8228		- array('role_restriction' => $role_restriction),
8229		- array('source_url' => $source_url),
8230		- array('%s'),
8231		- array('%s')
8232		- );
8233		- }
8234		-
8235		- if ($result !== false) {
8236		- $tag_updated++;
8237		- $updated_count++;
8238		- }
8239		- }
8240		-
8241		- if ($tag_updated > 0) {
8242		- $details[] = sprintf(
8243		- 'Tag "%s" (%s): %d posts updated',
8244		- $tag_slug,
8245		- $role_restriction,
8246		- $tag_updated
8247		- );
8248		- }
8249		- }
8250		-
8251		- wp_send_json_success(array(
8252		- 'message' => 'Bulk update completed',
8253		- 'updated_count' => $updated_count,
8254		- 'tags_processed' => count($mappings),
8255		- 'details' => $details
8256		- ));
8257		- exit;
8258		-}
8259		-
8260		-/**
8261		- * Handle tag changes on posts (when tags are added or removed)
8262		- */
8263		-public function handle_tag_change($object_id, $terms, $tt_ids, $taxonomy, $append, $old_tt_ids) {
8264		- // Only process post tags
8265		- if ($taxonomy !== 'post_tag') {
8266		- return;
8267		- }
8268		-
8269		- // Get tag-role mappings
8270		- $mappings = get_option('mxchat_tag_role_mappings', array());
8271		-
8272		- if (empty($mappings)) {
8273		- return;
8274		- }
8275		-
8276		- // Get the post's URL
8277		- $source_url = get_permalink($object_id);
8278		- if (!$source_url) {
8279		- return;
8280		- }
8281		-
8282		- // Determine the highest role restriction based on tags
8283		- $highest_role = 'public';
8284		- $role_hierarchy = array(
8285		- 'public' => 0,
8286		- 'logged_in' => 1,
8287		- 'subscriber' => 2,
8288		- 'contributor' => 3,
8289		- 'author' => 4,
8290		- 'editor' => 5,
8291		- 'administrator' => 6
8292		- );
8293		-
8294		- // Get all current tags for the post
8295		- $current_tags = wp_get_post_tags($object_id, array('fields' => 'slugs'));
8296		-
8297		- // Find the highest role restriction among the tags
8298		- foreach ($current_tags as $tag_slug) {
8299		- if (isset($mappings[$tag_slug])) {
8300		- $role = $mappings[$tag_slug];
8301		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
8302		- $highest_role = $role;
8303		- }
8304		- }
8305		- }
8306		-
8307		- // Update the role restriction in the database
8308		- global $wpdb;
8309		-
8310		- // Check if using Pinecone
8311		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
8312		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
8313		-
8314		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
8315		- // Update Pinecone role restriction
8316		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
8317		- $vector_id = md5($source_url);
8318		-
8319		- $wpdb->replace(
8320		- $roles_table,
8321		- array(
8322		- 'vector_id' => $vector_id,
8323		- 'role_restriction' => $highest_role,
8324		- 'updated_at' => current_time('mysql')
8325		- ),
8326		- array('%s', '%s', '%s')
8327		- );
8328		- } else {
8329		- // Update WordPress DB
8330		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
8331		-
8332		- $wpdb->update(
8333		- $table_name,
8334		- array('role_restriction' => $highest_role),
8335		- array('source_url' => $source_url),
8336		- array('%s'),
8337		- array('%s')
8338		- );
8339		- }
8340		-
8341		- // The entry's restriction just changed — keep the OpenAI Vector Store
8342		- // mirror consistent: non-public pulls the file (file_search has no
8343		- // per-role filtering), public re-mirrors it (plan 15b5c6).
8344		- if (class_exists('MxChat_Vectorstore_Manager')) {
8345		- MxChat_Vectorstore_Manager::handle_role_change($source_url, 'default', $highest_role);
8346		- }
8347		-}
8348		-
8349		-/**
8350		- * Apply role restriction after content is stored (for auto-sync)
8351		- */
8352		-public function apply_role_restriction_after_storage($post_id, $source_url) {
8353		- // Get tag-role mappings
8354		- $mappings = get_option('mxchat_tag_role_mappings', array());
8355		-
8356		- if (empty($mappings)) {
8357		- return;
8358		- }
8359		-
8360		- // Get all tags for the post
8361		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
8362		-
8363		- if (empty($post_tags)) {
8364		- return;
8365		- }
8366		-
8367		- // Determine the highest role restriction based on tags
8368		- $highest_role = 'public';
8369		- $role_hierarchy = array(
8370		- 'public' => 0,
8371		- 'logged_in' => 1,
8372		- 'subscriber' => 2,
8373		- 'contributor' => 3,
8374		- 'author' => 4,
8375		- 'editor' => 5,
8376		- 'administrator' => 6
8377		- );
8378		-
8379		- foreach ($post_tags as $tag_slug) {
8380		- if (isset($mappings[$tag_slug])) {
8381		- $role = $mappings[$tag_slug];
8382		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
8383		- $highest_role = $role;
8384		- }
8385		- }
8386		- }
8387		-
8388		- // If no restricted tags found, return (leave as public)
8389		- if ($highest_role === 'public') {
8390		- return;
8391		- }
8392		-
8393		- // Update the role restriction
8394		- global $wpdb;
8395		-
8396		- // Check if using Pinecone
8397		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
8398		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
8399		-
8400		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
8401		- // Update Pinecone role restriction
8402		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
8403		- $vector_id = md5($source_url);
8404		-
8405		- $wpdb->replace(
8406		- $roles_table,
8407		- array(
8408		- 'vector_id' => $vector_id,
8409		- 'role_restriction' => $highest_role,
8410		- 'updated_at' => current_time('mysql')
8411		- ),
8412		- array('%s', '%s', '%s')
8413		- );
8414		- } else {
8415		- // Update WordPress DB
8416		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
8417		-
8418		- $wpdb->update(
8419		- $table_name,
8420		- array('role_restriction' => $highest_role),
8421		- array('source_url' => $source_url),
8422		- array('%s'),
8423		- array('%s')
8424		- );
8425		- }
8426		-
8427		- // The entry's restriction just changed — keep the OpenAI Vector Store
8428		- // mirror consistent: non-public pulls the file (file_search has no
8429		- // per-role filtering), public re-mirrors it (plan 15b5c6).
8430		- if (class_exists('MxChat_Vectorstore_Manager')) {
8431		- MxChat_Vectorstore_Manager::handle_role_change($source_url, 'default', $highest_role);
8432		- }
8433		-}
8434		-
8435		-
8436	3485	// ========================================
8437	3486	// HELPER METHODS
8438	3487	// ========================================
8439	3488
		@@ -8500,981 +3549,8 @@
8500	3549	*/
8501	3550	private function mxchat_get_pinecone_manager() {
8502	3551	return MxChat_Pinecone_Manager::get_instance();
8503	3552	}
8504		-
8505		-
8506		- // ========================================
8507		-// DATABASE QUEUE TABLE MANAGEMENT
8508		-// ========================================
8509		-
8510		-/**
8511		- * Create queue table on plugin activation
8512		- * Call this from your plugin activation hook
8513		- */
8514		-public function mxchat_create_queue_table() {
8515		- global $wpdb;
8516		-
8517		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8518		- $charset_collate = $wpdb->get_charset_collate();
8519		-
8520		- $sql = "CREATE TABLE IF NOT EXISTS $table_name (
8521		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
8522		- queue_id varchar(64) NOT NULL,
8523		- item_type varchar(20) NOT NULL,
8524		- item_data longtext NOT NULL,
8525		- status varchar(20) NOT NULL DEFAULT 'pending',
8526		- bot_id varchar(50) NOT NULL DEFAULT 'default',
8527		- priority int(11) NOT NULL DEFAULT 0,
8528		- attempts int(11) NOT NULL DEFAULT 0,
8529		- max_attempts int(11) NOT NULL DEFAULT 3,
8530		- error_message text DEFAULT NULL,
8531		- created_at datetime NOT NULL,
8532		- started_at datetime DEFAULT NULL,
8533		- completed_at datetime DEFAULT NULL,
8534		- PRIMARY KEY (id),
8535		- KEY queue_id (queue_id),
8536		- KEY status (status),
8537		- KEY item_type (item_type),
8538		- KEY priority (priority)
8539		- ) $charset_collate;";
8540		-
8541		- require_once(ABSPATH . 'wp-admin/includes/upgrade.php');
8542		- dbDelta($sql);
8543		-
8544		- // Also create a meta table for queue metadata
8545		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
8546		-
8547		- $meta_sql = "CREATE TABLE IF NOT EXISTS $meta_table (
8548		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
8549		- queue_id varchar(64) NOT NULL,
8550		- meta_key varchar(255) NOT NULL,
8551		- meta_value longtext,
8552		- PRIMARY KEY (id),
8553		- KEY queue_id (queue_id),
8554		- KEY meta_key (meta_key)
8555		- ) $charset_collate;";
8556		-
8557		- dbDelta($meta_sql);
8558		-}
8559		-
8560		-/**
8561		- * Add items to the processing queue
8562		- *
8563		- * @param string $queue_id Unique identifier for this queue batch
8564		- * @param string $item_type Type of item (url, pdf_page)
8565		- * @param array $items Array of items to queue
8566		- * @param string $bot_id Bot ID for processing
8567		- * @return int Number of items queued
8568		- */
8569		-private function mxchat_add_to_queue($queue_id, $item_type, $items, $bot_id = 'default') {
8570		- global $wpdb;
8571		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8572		-
8573		- $queued_count = 0;
8574		- $priority = 0;
8575		-
8576		- foreach ($items as $item) {
8577		- $result = $wpdb->insert(
8578		- $table_name,
8579		- array(
8580		- 'queue_id' => $queue_id,
8581		- 'item_type' => $item_type,
8582		- 'item_data' => wp_json_encode($item),
8583		- 'status' => 'pending',
8584		- 'bot_id' => $bot_id,
8585		- 'priority' => $priority,
8586		- 'attempts' => 0,
8587		- 'max_attempts' => 3,
8588		- 'created_at' => current_time('mysql')
8589		- ),
8590		- array('%s', '%s', '%s', '%s', '%s', '%d', '%d', '%d', '%s')
8591		- );
8592		-
8593		- if ($result) {
8594		- $queued_count++;
8595		- }
8596		-
8597		- $priority++; // Process in order
8598		- }
8599		-
8600		- return $queued_count;
8601		-}
8602		-
8603		-/**
8604		- * Store queue metadata (total counts, source URL, etc.)
8605		- */
8606		-private function mxchat_set_queue_meta($queue_id, $meta_key, $meta_value) {
8607		- global $wpdb;
8608		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
8609		-
8610		- // Check if meta exists
8611		- $existing = $wpdb->get_var($wpdb->prepare(
8612		- "SELECT id FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
8613		- $queue_id,
8614		- $meta_key
8615		- ));
8616		-
8617		- if ($existing) {
8618		- // Update
8619		- $wpdb->update(
8620		- $meta_table,
8621		- array('meta_value' => maybe_serialize($meta_value)),
8622		- array('queue_id' => $queue_id, 'meta_key' => $meta_key),
8623		- array('%s'),
8624		- array('%s', '%s')
8625		- );
8626		- } else {
8627		- // Insert
8628		- $wpdb->insert(
8629		- $meta_table,
8630		- array(
8631		- 'queue_id' => $queue_id,
8632		- 'meta_key' => $meta_key,
8633		- 'meta_value' => maybe_serialize($meta_value)
8634		- ),
8635		- array('%s', '%s', '%s')
8636		- );
8637		- }
8638		-}
8639		-
8640		-/**
8641		- * Get queue metadata
8642		- */
8643		-private function mxchat_get_queue_meta($queue_id, $meta_key) {
8644		- global $wpdb;
8645		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
8646		-
8647		- $value = $wpdb->get_var($wpdb->prepare(
8648		- "SELECT meta_value FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
8649		- $queue_id,
8650		- $meta_key
8651		- ));
8652		-
8653		- return maybe_unserialize($value);
8654		-}
8655		-
8656		-// ========================================
8657		-// AJAX QUEUE PROCESSING HANDLERS
8658		-// ========================================
8659		-
8660		-/**
8661		- * AJAX: Get next item from queue to process
8662		- */
8663		-public function ajax_mxchat_get_next_queue_item() {
8664		- // Verify nonce and permissions
8665		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8666		-
8667		- if (!current_user_can('manage_options')) {
8668		- wp_send_json_error('Unauthorized access');
8669		- }
8670		-
8671		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8672		-
8673		- if (empty($queue_id)) {
8674		- wp_send_json_error('Missing queue ID');
8675		- }
8676		-
8677		- global $wpdb;
8678		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8679		-
8680		- // Get next pending item with retry logic for failed items
8681		- $next_item = $wpdb->get_row($wpdb->prepare(
8682		- "SELECT * FROM $table_name
8683		- WHERE queue_id = %s
8684		- AND status IN ('pending', 'failed')
8685		- AND attempts < max_attempts
8686		- ORDER BY priority ASC, id ASC
8687		- LIMIT 1",
8688		- $queue_id
8689		- ));
8690		-
8691		- if (!$next_item) {
8692		- // No more items - queue complete
8693		- wp_send_json_success(array(
8694		- 'complete' => true,
8695		- 'message' => 'Queue processing complete'
8696		- ));
8697		- }
8698		-
8699		- // Mark item as processing
8700		- $wpdb->update(
8701		- $table_name,
8702		- array(
8703		- 'status' => 'processing',
8704		- 'started_at' => current_time('mysql'),
8705		- 'attempts' => $next_item->attempts + 1
8706		- ),
8707		- array('id' => $next_item->id),
8708		- array('%s', '%s', '%d'),
8709		- array('%d')
8710		- );
8711		-
8712		- wp_send_json_success(array(
8713		- 'complete' => false,
8714		- 'item' => array(
8715		- 'id' => $next_item->id,
8716		- 'type' => $next_item->item_type,
8717		- 'data' => json_decode($next_item->item_data, true),
8718		- 'bot_id' => $next_item->bot_id,
8719		- 'attempt' => $next_item->attempts + 1
8720		- )
8721		- ));
8722		-}
8723		-
8724		-/**
8725		- * AJAX: Process a single queue item
8726		- */
8727		-public function ajax_mxchat_process_queue_item() {
8728		- // Verify nonce and permissions
8729		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8730		-
8731		- if (!current_user_can('manage_options')) {
8732		- wp_send_json_error('Unauthorized access');
8733		- }
8734		-
8735		- $item_id = isset($_POST['item_id']) ? absint($_POST['item_id']) : 0;
8736		- $item_type = isset($_POST['item_type']) ? sanitize_text_field($_POST['item_type']) : '';
8737		- $item_data = isset($_POST['item_data']) ? $_POST['item_data'] : array();
8738		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
8739		-
8740		- if (empty($item_id) || empty($item_type)) {
8741		- wp_send_json_error('Missing item data');
8742		- }
8743		-
8744		- global $wpdb;
8745		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8746		-
8747		- // Process based on item type
8748		- try {
8749		- set_time_limit(60); // Give processing 60 seconds
8750		-
8751		- $result = false;
8752		- $error_message = '';
8753		-
8754		- // Read item directly from DB to get queue_id and preserve special chars in item_data
8755		- // (POST round-trip through JS mangles characters like apostrophes in URLs)
8756		- $db_item = $wpdb->get_row($wpdb->prepare(
8757		- "SELECT queue_id, item_data FROM $table_name WHERE id = %d",
8758		- $item_id
8759		- ));
8760		- $item_queue_id = $db_item ? $db_item->queue_id : '';
8761		- if ($db_item && !empty($db_item->item_data)) {
8762		- $db_data = json_decode($db_item->item_data, true);
8763		- if (is_array($db_data)) {
8764		- $item_data = $db_data;
8765		- }
8766		- }
8767		-
8768		- switch ($item_type) {
8769		- case 'url':
8770		- $result = $this->mxchat_process_queue_url($item_data, $bot_id, $item_queue_id);
8771		- break;
8772		-
8773		- case 'pdf_page':
8774		- $result = $this->mxchat_process_queue_pdf_page($item_data, $bot_id);
8775		- break;
8776		-
8777		- default:
8778		- throw new Exception('Unknown item type: ' . $item_type);
8779		- }
8780		-
8781		- if (is_wp_error($result)) {
8782		- $error_code = $result->get_error_code();
8783		- // Content errors (empty page, sanitization) are permanent — retrying won't help
8784		- $permanent_codes = array('empty_page', 'empty_after_sanitization', 'no_api_key', 'page_not_found');
8785		- if (in_array($error_code, $permanent_codes)) {
8786		- // Mark as permanently failed — set attempts = max_attempts so it won't be retried
8787		- $current_item = $wpdb->get_row($wpdb->prepare(
8788		- "SELECT max_attempts FROM $table_name WHERE id = %d", $item_id
8789		- ));
8790		- $wpdb->update(
8791		- $table_name,
8792		- array(
8793		- 'status' => 'failed',
8794		- 'error_message' => $result->get_error_message(),
8795		- 'attempts' => $current_item ? $current_item->max_attempts : 3
8796		- ),
8797		- array('id' => $item_id),
8798		- array('%s', '%s', '%d'),
8799		- array('%d')
8800		- );
8801		- wp_send_json_error(array(
8802		- 'message' => $result->get_error_message(),
8803		- 'permanent_failure' => true,
8804		- 'item_id' => $item_id
8805		- ));
8806		- return;
8807		- }
8808		- throw new Exception($result->get_error_message());
8809		- }
8810		-
8811		- if ($result === false) {
8812		- throw new Exception('Processing returned false - item may be empty or invalid');
8813		- }
8814		-
8815		- // Mark as completed
8816		- $wpdb->update(
8817		- $table_name,
8818		- array(
8819		- 'status' => 'completed',
8820		- 'completed_at' => current_time('mysql'),
8821		- 'error_message' => null
8822		- ),
8823		- array('id' => $item_id),
8824		- array('%s', '%s', '%s'),
8825		- array('%d')
8826		- );
8827		-
8828		- wp_send_json_success(array(
8829		- 'processed' => true,
8830		- 'item_id' => $item_id,
8831		- 'message' => 'Item processed successfully'
8832		- ));
8833		-
8834		- } catch (Exception $e) {
8835		- $error_message = $e->getMessage();
8836		-
8837		- // Get current attempt count
8838		- $item = $wpdb->get_row($wpdb->prepare(
8839		- "SELECT attempts, max_attempts FROM $table_name WHERE id = %d",
8840		- $item_id
8841		- ));
8842		-
8843		- // Check if we've exhausted retries
8844		- if ($item && $item->attempts >= $item->max_attempts) {
8845		- // Permanently failed
8846		- $wpdb->update(
8847		- $table_name,
8848		- array(
8849		- 'status' => 'failed',
8850		- 'error_message' => $error_message
8851		- ),
8852		- array('id' => $item_id),
8853		- array('%s', '%s'),
8854		- array('%d')
8855		- );
8856		-
8857		- wp_send_json_error(array(
8858		- 'message' => 'Item failed after maximum attempts: ' . $error_message,
8859		- 'permanent_failure' => true,
8860		- 'item_id' => $item_id
8861		- ));
8862		- } else {
8863		- // Mark for retry
8864		- $wpdb->update(
8865		- $table_name,
8866		- array(
8867		- 'status' => 'failed',
8868		- 'error_message' => $error_message
8869		- ),
8870		- array('id' => $item_id),
8871		- array('%s', '%s'),
8872		- array('%d')
8873		- );
8874		-
8875		- wp_send_json_error(array(
8876		- 'message' => 'Item processing failed, will retry: ' . $error_message,
8877		- 'can_retry' => true,
8878		- 'item_id' => $item_id,
8879		- 'attempts' => $item ? $item->attempts : 0
8880		- ));
8881		- }
8882		- }
8883		-}
8884		-
8885		-/**
8886		- * Process a URL from the queue
8887		- */
8888		-private function mxchat_process_queue_url($item_data, $bot_id = 'default', $queue_id = '') {
8889		- $url = isset($item_data['url']) ? $item_data['url'] : '';
8890		-
8891		- if (empty($url)) {
8892		- return new WP_Error('invalid_url', 'URL is empty');
8893		- }
8894		-
8895		- // Get bot-specific embedding decision early (needed for both paths) —
8896		- // custom-provider-aware (plan cbd5fd). Error code preserved.
8897		- $bot_options = $this->get_bot_options($bot_id);
8898		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
8899		-
8900		- $preflight = MxChat_Utils::embedding_preflight($options);
8901		- if (!$preflight['ok']) {
8902		- return new WP_Error('no_api_key', $preflight['reason'] . ' (bot: ' . $bot_id . ')');
8903		- }
8904		- $api_key = $preflight['api_key'];
8905		-
8906		- // Check if this is a WooCommerce product URL and WooCommerce is active
8907		- $is_product_url = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
8908		- $content_type = $is_product_url ? 'product' : 'url';
8909		-
8910		- // Try to get WooCommerce product data if it's a product URL
8911		- if ($is_product_url && class_exists('WooCommerce')) {
8912		- $product_content = $this->mxchat_extract_woocommerce_product_content($url);
8913		-
8914		- if (!empty($product_content)) {
8915		- // Successfully extracted WooCommerce product data with pricing
8916		- $result = MxChat_Utils::submit_content_to_db(
8917		- $product_content,
8918		- $url,
8919		- $api_key,
8920		- null,
8921		- $bot_id,
8922		- 'product'
8923		- );
8924		- return $result;
8925		- }
8926		- // If WooCommerce extraction failed, fall through to HTML extraction
8927		- }
8928		-
8929		- // Fetch URL content (fallback for non-products or when WooCommerce extraction fails)
8930		- $is_likely_pdf = (strtolower(pathinfo(parse_url($url, PHP_URL_PATH) ?: '', PATHINFO_EXTENSION)) === 'pdf');
8931		- $response = wp_remote_get($url, array(
8932		- 'timeout' => $is_likely_pdf ? 120 : 30,
8933		- 'redirection' => 5,
8934		- 'user-agent' => mxchat_ingest_user_agent(),
8935		- ));
8936		-
8937		- if (is_wp_error($response)) {
8938		- return $response;
8939		- }
8940		-
8941		- $response_code = wp_remote_retrieve_response_code($response);
8942		- if ($response_code !== 200) {
8943		- return new WP_Error('http_error', 'HTTP ' . $response_code . ' error for: ' . $url);
8944		- }
8945		-
8946		- // Check if URL is a PDF — expand into per-page queue items using the standard PDF pipeline
8947		- if ($this->mxchat_is_pdf_url($url, $response)) {
8948		- return $this->mxchat_expand_pdf_to_queue($url, $response, $bot_id, $queue_id);
8949		- }
8950		-
8951		- $html = wp_remote_retrieve_body($response);
8952		-
8953		- if (empty($html)) {
8954		- return new WP_Error('empty_response', 'Empty response body');
8955		- }
8956		-
8957		- // Extract and sanitize content
8958		- $content = $this->mxchat_extract_main_content($html);
8959		- $sanitized = $this->mxchat_sanitize_content_for_api($content);
8960		-
8961		- if (empty($sanitized)) {
8962		- // Not an error - just no content found (maybe a redirect or empty page)
8963		- return false;
8964		- }
8965		-
8966		- // Submit to database with content_type
8967		- $result = MxChat_Utils::submit_content_to_db(
8968		- $sanitized,
8969		- $url,
8970		- $api_key,
8971		- null,
8972		- $bot_id,
8973		- $content_type
8974		- );
8975		-
8976		- return $result;
8977		-}
8978		-
8979		-/**
8980		- * Expand a PDF URL into per-page queue items using the standard PDF pipeline.
8981		- * Called when a sitemap URL turns out to be a PDF — downloads, parses page count,
8982		- * and adds pdf_page items to the same queue so they process with full progress tracking.
8983		- */
8984		-private function mxchat_expand_pdf_to_queue($pdf_url, $response, $bot_id = 'default', $queue_id = '') {
8985		- set_time_limit(120); // PDFs need extra time for download + parsing
8986		-
8987		- $upload_dir = wp_upload_dir();
8988		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
8989		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
8990		-
8991		- $response_body = wp_remote_retrieve_body($response);
8992		- if (empty($response_body)) {
8993		- return new WP_Error('empty_pdf', 'Empty PDF response for: ' . $pdf_url);
8994		- }
8995		-
8996		- if (!wp_mkdir_p(dirname($pdf_path))) {
8997		- return new WP_Error('dir_error', 'Failed to create upload directory');
8998		- }
8999		-
9000		- file_put_contents($pdf_path, $response_body);
9001		-
9002		- if (!file_exists($pdf_path)) {
9003		- return new WP_Error('save_error', 'Failed to save PDF file');
9004		- }
9005		-
9006		- try {
9007		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
9008		-
9009		- if ($total_pages === false || $total_pages < 1) {
9010		- wp_delete_file($pdf_path);
9011		- return new WP_Error('no_pages', 'PDF has no pages: ' . $pdf_url);
9012		- }
9013		-
9014		- // Build per-page items identical to mxchat_handle_pdf_for_knowledge_base
9015		- $pages = array();
9016		- for ($i = 1; $i <= $total_pages; $i++) {
9017		- $pages[] = array(
9018		- 'pdf_path' => $pdf_path,
9019		- 'pdf_url' => $pdf_url,
9020		- 'page_number' => $i,
9021		- 'total_pages' => $total_pages
9022		- );
9023		- }
9024		-
9025		- // Add pdf_page items to the SAME queue so the JS picks them up automatically
9026		- if (!empty($queue_id)) {
9027		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
9028		- } else {
9029		- // Fallback: create a new PDF queue (shouldn't happen in sitemap flow)
9030		- $new_queue_id = 'pdf_' . md5($pdf_url . time());
9031		- $queued_count = $this->mxchat_add_to_queue($new_queue_id, 'pdf_page', $pages, $bot_id);
9032		- $this->mxchat_set_queue_meta($new_queue_id, 'source_url', $pdf_url);
9033		- $this->mxchat_set_queue_meta($new_queue_id, 'queue_type', 'pdf');
9034		- $this->mxchat_set_queue_meta($new_queue_id, 'total_items', $total_pages);
9035		- $this->mxchat_set_queue_meta($new_queue_id, 'bot_id', $bot_id);
9036		- $this->mxchat_set_queue_meta($new_queue_id, 'pdf_path', $pdf_path);
9037		- $this->mxchat_set_queue_meta($new_queue_id, 'created_at', current_time('mysql'));
9038		- }
9039		-
9040		- if ($queued_count === 0) {
9041		- wp_delete_file($pdf_path);
9042		- return new WP_Error('queue_error', 'Failed to add PDF pages to queue');
9043		- }
9044		-
9045		- // Return true so the original URL item is marked complete
9046		- // The new pdf_page items will be processed in subsequent batches
9047		- return true;
9048		-
9049		- } catch (Exception $e) {
9050		- if (file_exists($pdf_path)) {
9051		- wp_delete_file($pdf_path);
9052		- }
9053		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
9054		- }
9055		-}
9056		-
9057		-/**
9058		- * Legacy: Process a PDF URL inline during sitemap queue processing.
9059		- * @deprecated Use mxchat_expand_pdf_to_queue instead — kept for reference only.
9060		- */
9061		-private function mxchat_process_pdf_url_inline($pdf_url, $response, $api_key, $bot_id = 'default') {
9062		- set_time_limit(120); // PDFs need more time — downloading + parsing all pages
9063		-
9064		- $upload_dir = wp_upload_dir();
9065		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
9066		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
9067		-
9068		- $response_body = wp_remote_retrieve_body($response);
9069		- if (empty($response_body)) {
9070		- return new WP_Error('empty_pdf', 'Empty PDF response');
9071		- }
9072		-
9073		- if (!wp_mkdir_p(dirname($pdf_path))) {
9074		- return new WP_Error('dir_error', 'Failed to create upload directory');
9075		- }
9076		-
9077		- file_put_contents($pdf_path, $response_body);
9078		-
9079		- if (!file_exists($pdf_path)) {
9080		- return new WP_Error('save_error', 'Failed to save PDF file');
9081		- }
9082		-
9083		- try {
9084		- mxchat_load_pdf_parser();
9085		- $parser = new \Smalot\PdfParser\Parser();
9086		- $pdf = $parser->parseFile($pdf_path);
9087		- $pages = $pdf->getPages();
9088		- $total_pages = count($pages);
9089		-
9090		- if ($total_pages < 1) {
9091		- wp_delete_file($pdf_path);
9092		- return new WP_Error('no_pages', 'PDF has no pages');
9093		- }
9094		-
9095		- $processed = 0;
9096		- $skipped_pages = array();
9097		-
9098		- for ($i = 0; $i < $total_pages; $i++) {
9099		- $page_num = $i + 1;
9100		- $text = $pages[$i]->getText();
9101		- $text = MxChat_Utils::normalize_pdf_rtl($text, 'kb_pdf_import page ' . $page_num);
9102		- if (empty($text)) {
9103		- $skipped_pages[] = 'Page ' . $page_num . ': No text could be extracted — page may contain only images, links, or non-standard encoding';
9104		- continue;
9105		- }
9106		-
9107		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
9108		- if (empty($sanitized)) {
9109		- $skipped_pages[] = 'Page ' . $page_num . ': Text was extracted but contained only special characters, control codes, or unsupported content';
9110		- continue;
9111		- }
9112		-
9113		- $metadata = array(
9114		- 'document_type' => 'pdf',
9115		- 'total_pages' => $total_pages,
9116		- 'current_page' => $page_num,
9117		- 'source_url' => $pdf_url,
9118		- );
9119		-
9120		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
9121		- $page_url = esc_url($pdf_url . '#page=' . $page_num);
9122		-
9123		- MxChat_Utils::submit_content_to_db(
9124		- $content_with_metadata,
9125		- $page_url,
9126		- $api_key,
9127		- null,
9128		- $bot_id,
9129		- 'pdf'
9130		- );
9131		-
9132		- $processed++;
9133		- }
9134		-
9135		- // Clean up the temp PDF file
9136		- wp_delete_file($pdf_path);
9137		-
9138		- if (!empty($skipped_pages)) {
9139		- error_log('MxChat PDF: Skipped ' . count($skipped_pages) . ' of ' . $total_pages . ' pages: ' . implode('; ', $skipped_pages));
9140		- }
9141		-
9142		- return $processed > 0 ? true : false;
9143		-
9144		- } catch (Exception $e) {
9145		- if (file_exists($pdf_path)) {
9146		- wp_delete_file($pdf_path);
9147		- }
9148		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
9149		- }
9150		-}
9151		-
9152		-/**
9153		- * Extract WooCommerce product content including pricing
9154		- *
9155		- * @param string $url The product URL
9156		- * @return string|false Product content with pricing, or false if not found
9157		- */
9158		-private function mxchat_extract_woocommerce_product_content($url) {
9159		- // Try to get product ID from URL
9160		- $product_id = url_to_postid($url);
9161		-
9162		- // If url_to_postid fails, try to extract from URL pattern
9163		- if (!$product_id) {
9164		- $product_slug = '';
9165		-
9166		- // Handle pretty permalinks: /product/product-name/
9167		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
9168		- $product_slug = $matches[1];
9169		- }
9170		-
9171		- if (!empty($product_slug)) {
9172		- $product_post = get_page_by_path($product_slug, OBJECT, 'product');
9173		- if ($product_post) {
9174		- $product_id = $product_post->ID;
9175		- }
9176		- }
9177		- }
9178		-
9179		- if (!$product_id) {
9180		- return false;
9181		- }
9182		-
9183		- // Get WooCommerce product object
9184		- $product = wc_get_product($product_id);
9185		-
9186		- if (!$product) {
9187		- return false;
9188		- }
9189		-
9190		- // Build product content via the shared WC-object assembler (a3d60c) — same
9191		- // body as the auto-sync product writer, so the two paths can never drift.
9192		- $content = $this->mxchat_prepare_product_content_for_indexing($product);
9193		-
9194		- return $this->mxchat_sanitize_content_for_api($content);
9195		-}
9196		-
9197		-/**
9198		- * Process a PDF page from the queue
9199		- */
9200		-private function mxchat_process_queue_pdf_page($item_data, $bot_id = 'default') {
9201		- $pdf_path = isset($item_data['pdf_path']) ? $item_data['pdf_path'] : '';
9202		- $pdf_url = isset($item_data['pdf_url']) ? $item_data['pdf_url'] : '';
9203		- $page_number = isset($item_data['page_number']) ? absint($item_data['page_number']) : 0;
9204		- $total_pages = isset($item_data['total_pages']) ? absint($item_data['total_pages']) : 0;
9205		-
9206		- if (empty($pdf_path) || !file_exists($pdf_path)) {
9207		- return new WP_Error('pdf_not_found', 'PDF file not found: ' . $pdf_path);
9208		- }
9209		-
9210		- if ($page_number < 1) {
9211		- return new WP_Error('invalid_page', 'Invalid page number');
9212		- }
9213		-
9214		- try {
9215		- mxchat_load_pdf_parser();
9216		- $parser = new \Smalot\PdfParser\Parser();
9217		- $pdf = $parser->parseFile($pdf_path);
9218		- $pages = $pdf->getPages();
9219		-
9220		- if (!isset($pages[$page_number - 1])) {
9221		- return new WP_Error('page_not_found', 'Page ' . $page_number . ' not found in PDF');
9222		- }
9223		-
9224		- $text = $pages[$page_number - 1]->getText();
9225		- $text = MxChat_Utils::normalize_pdf_rtl($text, 'kb_pdf_page page ' . $page_number);
9226		-
9227		- if (empty($text)) {
9228		- return new WP_Error('empty_page', 'Page ' . $page_number . ': No text could be extracted — page may contain only images, links, or non-standard encoding');
9229		- }
9230		-
9231		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
9232		-
9233		- if (empty($sanitized)) {
9234		- return new WP_Error('empty_after_sanitization', 'Page ' . $page_number . ': Text was extracted but contained only special characters, control codes, or unsupported content that was removed during cleanup');
9235		- }
9236		-
9237		- // Create metadata
9238		- $metadata = array(
9239		- 'document_type' => 'pdf',
9240		- 'total_pages' => $total_pages,
9241		- 'current_page' => $page_number,
9242		- 'source_url' => $pdf_url
9243		- );
9244		-
9245		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
9246		- $page_url = esc_url($pdf_url . "#page=" . $page_number);
9247		-
9248		- // Get bot-specific embedding decision — custom-provider-aware
9249		- // (plan cbd5fd). Error code preserved.
9250		- $bot_options = $this->get_bot_options($bot_id);
9251		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
9252		-
9253		- $preflight = MxChat_Utils::embedding_preflight($options);
9254		- if (!$preflight['ok']) {
9255		- return new WP_Error('no_api_key', $preflight['reason'] . ' (bot: ' . $bot_id . ')');
9256		- }
9257		- $api_key = $preflight['api_key'];
9258		-
9259		- // Submit to database - UPDATED 2.5.6: Added content_type 'pdf'
9260		- $result = MxChat_Utils::submit_content_to_db(
9261		- $content_with_metadata,
9262		- $page_url,
9263		- $api_key,
9264		- null,
9265		- $bot_id,
9266		- 'pdf'
9267		- );
9268		-
9269		- return $result;
9270		-
9271		- } catch (Exception $e) {
9272		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
9273		- }
9274		-}
9275		-
9276		-/**
9277		- * AJAX: Get queue processing status
9278		- */
9279		-public function ajax_mxchat_get_queue_status() {
9280		- // Verify nonce and permissions
9281		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
9282		-
9283		- if (!current_user_can('manage_options')) {
9284		- wp_send_json_error('Unauthorized access');
9285		- }
9286		-
9287		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
9288		-
9289		- if (empty($queue_id)) {
9290		- wp_send_json_error('Missing queue ID');
9291		- }
9292		-
9293		- global $wpdb;
9294		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
9295		-
9296		- // Get counts by status
9297		- $counts = $wpdb->get_results($wpdb->prepare(
9298		- "SELECT status, COUNT(*) as count
9299		- FROM $table_name
9300		- WHERE queue_id = %s
9301		- GROUP BY status",
9302		- $queue_id
9303		- ), OBJECT_K);
9304		-
9305		- $total = 0;
9306		- $completed = 0;
9307		- $failed = 0;
9308		- $processing = 0;
9309		- $pending = 0;
9310		-
9311		- foreach ($counts as $status => $data) {
9312		- $count = absint($data->count);
9313		- $total += $count;
9314		-
9315		- switch ($status) {
9316		- case 'completed':
9317		- $completed = $count;
9318		- break;
9319		- case 'failed':
9320		- $failed = $count;
9321		- break;
9322		- case 'processing':
9323		- $processing = $count;
9324		- break;
9325		- case 'pending':
9326		- $pending = $count;
9327		- break;
9328		- }
9329		- }
9330		-
9331		- // Calculate percentage
9332		- $percentage = $total > 0 ? round((($completed + $failed) / $total) * 100) : 0;
9333		-
9334		- // Get failed items details (include all failed items, not just those that exhausted retries)
9335		- $failed_items = array();
9336		- if ($failed > 0) {
9337		- $failed_items = $wpdb->get_results($wpdb->prepare(
9338		- "SELECT item_type, item_data, error_message, attempts
9339		- FROM $table_name
9340		- WHERE queue_id = %s
9341		- AND status = 'failed'
9342		- ORDER BY id DESC
9343		- LIMIT 50",
9344		- $queue_id
9345		- ));
9346		- }
9347		-
9348		- // Get queue metadata
9349		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
9350		- $queue_type = $this->mxchat_get_queue_meta($queue_id, 'queue_type');
9351		-
9352		- // Determine if queue is complete
9353		- $is_complete = ($pending === 0 && $processing === 0);
9354		-
9355		- wp_send_json_success(array(
9356		- 'queue_id' => $queue_id,
9357		- 'queue_type' => $queue_type,
9358		- 'source_url' => $source_url,
9359		- 'total' => $total,
9360		- 'completed' => $completed,
9361		- 'failed' => $failed,
9362		- 'processing' => $processing,
9363		- 'pending' => $pending,
9364		- 'percentage' => $percentage,
9365		- 'is_complete' => $is_complete,
9366		- 'failed_items' => $failed_items,
9367		- 'status' => $is_complete ? 'complete' : 'processing'
9368		- ));
9369		-}
9370		-
9371		-/**
9372		- * AJAX: Clear completed queue
9373		- */
9374		-public function ajax_mxchat_clear_queue() {
9375		- // Verify nonce and permissions
9376		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
9377		-
9378		- if (!current_user_can('manage_options')) {
9379		- wp_send_json_error('Unauthorized access');
9380		- }
9381		-
9382		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
9383		-
9384		- if (empty($queue_id)) {
9385		- wp_send_json_error('Missing queue ID');
9386		- }
9387		-
9388		- global $wpdb;
9389		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
9390		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
9391		-
9392		- // Delete queue items
9393		- $wpdb->delete(
9394		- $table_name,
9395		- array('queue_id' => $queue_id),
9396		- array('%s')
9397		- );
9398		-
9399		- // Delete queue metadata
9400		- $wpdb->delete(
9401		- $meta_table,
9402		- array('queue_id' => $queue_id),
9403		- array('%s')
9404		- );
9405		-
9406		- wp_send_json_success(array(
9407		- 'message' => 'Queue cleared successfully'
9408		- ));
9409		-}
9410		-
9411		-/**
9412		- * AJAX: Retry failed items in queue
9413		- */
9414		-public function ajax_mxchat_retry_failed() {
9415		- // Verify nonce and permissions
9416		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
9417		-
9418		- if (!current_user_can('manage_options')) {
9419		- wp_send_json_error('Unauthorized access');
9420		- }
9421		-
9422		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
9423		-
9424		- if (empty($queue_id)) {
9425		- wp_send_json_error('Missing queue ID');
9426		- }
9427		-
9428		- global $wpdb;
9429		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
9430		-
9431		- // Reset failed items to pending and reset attempt count
9432		- $updated = $wpdb->update(
9433		- $table_name,
9434		- array(
9435		- 'status' => 'pending',
9436		- 'attempts' => 0,
9437		- 'error_message' => null
9438		- ),
9439		- array(
9440		- 'queue_id' => $queue_id,
9441		- 'status' => 'failed'
9442		- ),
9443		- array('%s', '%d', '%s'),
9444		- array('%s', '%s')
9445		- );
9446		-
9447		- wp_send_json_success(array(
9448		- 'message' => 'Reset ' . $updated . ' failed items for retry',
9449		- 'reset_count' => $updated
9450		- ));
9451		-}
9452		-
9453		-
9454		-public function ajax_mxchat_mark_queue_complete() {
9455		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
9456		-
9457		- if (!current_user_can('manage_options')) {
9458		- wp_send_json_error('Unauthorized access');
9459		- }
9460		-
9461		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
9462		-
9463		- if (empty($queue_id)) {
9464		- wp_send_json_error('Missing queue ID');
9465		- }
9466		-
9467		- // Clear active queue transients
9468		- if (strpos($queue_id, 'sitemap_') === 0) {
9469		- delete_transient('mxchat_active_queue_sitemap');
9470		- } else if (strpos($queue_id, 'pdf_') === 0) {
9471		- delete_transient('mxchat_active_queue_pdf');
9472		- }
9473		-
9474		- wp_send_json_success(array('message' => 'Queue marked as complete'));
9475		-}
9476		-
9477	3553
9478	3554	// ========================================
9479	3555	// STATIC ACCESS METHODS
9480	3556	// ========================================