← All changes | admin/class-knowledge-manager.php +1661 -7533 3.2.19 → 2.2.3 View file →

		@@ -9,110 +9,55 @@
9	9	exit; // Exit if accessed directly
10	10	}
11	11
12	12	class MxChat_Knowledge_Manager {
13		-
	13	+
14	14	private $options;
15		-
16		- // Post IDs whose vectors were already deleted by mxchat_handle_status_transition this
17		- // request, so the transient-based branch in mxchat_handle_post_update can skip the
18		- // redundant (idempotent but network-visible) second deletion.
19		- private $transition_deleted_posts = array();
20		-
21		- // Post IDs already INDEXED by mxchat_handle_status_transition's arrival edge this
22		- // request. Normal editor publishes fire transition_post_status first, then
23		- // post_updated — without this guard every editor publish would embed twice.
24		- private $transition_indexed_posts = array();
25		-
26		- // Post IDs core has announced an in-flight UPDATE for. pre_post_update fires only
27		- // inside wp_insert_post's update branch and always before wp_transition_post_status,
28		- // so this is an exact "a post_updated is coming later this request" signal — which is
29		- // what makes it safe to arm transition_indexed_posts (plan a664f3).
30		- private $pending_post_update = array();
31		-
	15	+
32	16	/**
33	17	* Constructor - Register hooks for content processing
34	18	*/
35		-public function __construct() {
36		- $this->options = get_option('mxchat_options', array());
37		- $this->mxchat_init_hooks();
	19	+ public function __construct() {
	20	+ $this->options = get_option('mxchat_options', array());
	21	+ $this->mxchat_init_hooks();
	22	+ }
38	23
39		- $this->mxchat_init_role_hooks();
40		-}
41		-
42		-/**
43		- * Initialize WordPress hooks for content processing
44		- *
45		- */
46		-private function mxchat_init_hooks() {
47		- // Admin post handlers for form submissions
48		- add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
49		- add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
50		- add_action('admin_post_mxchat_submit_pdf_file', array($this, 'mxchat_handle_pdf_file_submission'));
51		- add_action('admin_post_mxchat_submit_youtube', array($this, 'mxchat_handle_youtube_submission'));
52		- add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
53		-
54		- // AJAX handlers for real-time processing and status updates
55		- add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
56		- add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status'));
57		- add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
58		- add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
59		- add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
60		- add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
61		- add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
62		- add_action('wp_ajax_mxchat_delete_chunks_by_url', array($this, 'ajax_mxchat_delete_chunks_by_url'));
63		- add_action('wp_ajax_mxchat_delete_wordpress_prompt', array($this, 'ajax_mxchat_delete_wordpress_prompt'));
64		- add_action('wp_ajax_mxchat_bulk_delete_knowledge', array($this, 'ajax_mxchat_bulk_delete_knowledge'));
65		- add_action('wp_ajax_mxchat_update_role_restriction', array($this, 'ajax_mxchat_update_role_restriction'));
66		-
67		- // Queue-based processing AJAX handlers
68		- add_action('wp_ajax_mxchat_get_next_queue_item', array($this, 'ajax_mxchat_get_next_queue_item'));
69		- add_action('wp_ajax_mxchat_process_queue_item', array($this, 'ajax_mxchat_process_queue_item'));
70		- add_action('wp_ajax_mxchat_get_queue_status', array($this, 'ajax_mxchat_get_queue_status'));
71		- add_action('wp_ajax_mxchat_clear_queue', array($this, 'ajax_mxchat_clear_queue'));
72		- add_action('wp_ajax_mxchat_retry_failed', array($this, 'ajax_mxchat_retry_failed'));
73		- add_action('wp_ajax_mxchat_get_recent_entries', array($this, 'ajax_mxchat_get_recent_entries'));
74		- add_action('wp_ajax_mxchat_detect_sitemaps', array($this, 'ajax_mxchat_detect_sitemaps'));
75		- add_action('wp_ajax_mxchat_refresh_pinecone_entries', array($this, 'ajax_mxchat_refresh_pinecone_entries'));
76		- add_action('wp_ajax_mxchat_paginate_entries', array($this, 'ajax_mxchat_paginate_entries'));
77		- add_action('wp_ajax_mxchat_get_entry_content', array($this, 'ajax_mxchat_get_entry_content'));
78		- add_action('wp_ajax_mxchat_save_entry_content', array($this, 'ajax_mxchat_save_entry_content'));
79		- add_action('wp_ajax_mxchat_inspect_entry', array($this, 'ajax_mxchat_inspect_entry'));
	24	+ /**
	25	+ * Initialize WordPress hooks for content processing
	26	+ */
	27	+ private function mxchat_init_hooks() {
	28	+ // Admin post handlers for form submissions
	29	+ add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
	30	+ add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
	31	+ add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
	32	+
	33	+ // AJAX handlers for real-time processing and status updates
	34	+ add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
	35	+ add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status')); // NEW
	36	+ add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
	37	+ add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
	38	+ add_action('wp_ajax_mxchat_manual_batch_process', array($this, 'ajax_manual_batch_process'));
	39	+ add_action('mxchat_delete_content', array($this, 'mxchat_delete_from_pinecone_by_url'), 10, 1);
80	40
81		- // WordPress post management hooks
82		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
83		- add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
84		- add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
85		- add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
86		- // Authoritative unpublish detection: core hands this hook the REAL previous status, so
87		- // removal no longer depends on the mxchat_prev_status_* transients (evictable by persistent
88		- // object caches, never written by paths that bypass wp_update_post, e.g. plugins flipping
89		- // post_status directly and calling wp_transition_post_status themselves).
90		- add_action('transition_post_status', array($this, 'mxchat_handle_status_transition'), 10, 3);
	41	+
	42	+ // Cron handlers for background processing
	43	+ add_action('mxchat_process_sitemap_urls', array($this, 'mxchat_process_sitemap_urls_cron'), 10, 5);
	44	+ add_action('mxchat_process_pdf_pages', array($this, 'mxchat_process_pdf_pages_cron'), 10, 5);
	45	+
	46	+ // WordPress post management hooks
	47	+ add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
	48	+ add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
	49	+ add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
	50	+ add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
91	51
92		- // ACF hook - fires AFTER ACF fields are saved, ensuring ACF data is available
93		- // Priority 20 to run after ACF's own save (which runs at priority 10)
94		- add_action('acf/save_post', array($this, 'mxchat_handle_acf_save'), 20);
95		-
96		- // One-time cleanup for vectors orphaned by unpublishes that predate the
97		- // transition_post_status handler (plan 816fb1): wp mxchat prune-unpublished
98		- if (defined('WP_CLI') && WP_CLI) {
99		- WP_CLI::add_command('mxchat prune-unpublished', array($this, 'cli_prune_unpublished'));
100		- // In-place repair for RTL KB rows imported in visual order before the
101		- // 32bf9e normalizer existed: wp mxchat rtl-repair (plan d1e6f7)
102		- WP_CLI::add_command('mxchat rtl-repair', array($this, 'cli_rtl_repair'));
	52	+ // WooCommerce product hooks (if WooCommerce is active)
	53	+ if (class_exists('WooCommerce')) {
	54	+ add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
	55	+ add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
	56	+ add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
	57	+ }
	58	+
103	59	}
104		-
105		- add_action('wp_ajax_mxchat_mark_queue_complete', array($this, 'ajax_mxchat_mark_queue_complete'));
106		-
107		- // WooCommerce product hooks (if WooCommerce is active)
108		- if (class_exists('WooCommerce')) {
109		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
110		- add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
111		- add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
112		- add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
113		- }
114		-}
115	60
116	61	/**
117	62	* Get current options (refreshed)
118	63	*/
		@@ -122,9 +67,315 @@
122	67	}
123	68	return $this->options;
124	69	}
125	70
	71	+
	72	+ /**
	73	+ * Handle manual batch processing via AJAX
	74	+ */
	75	+public function ajax_manual_batch_process() {
	76	+ try {
	77	+ // Verify nonce and permissions
	78	+ check_ajax_referer('mxchat_status_nonce', 'nonce');
	79	+
	80	+ if (!current_user_can('manage_options')) {
	81	+ wp_send_json_error('Unauthorized access');
	82	+ }
	83	+
	84	+ $process_type = sanitize_text_field($_POST['process_type'] ?? '');
	85	+ $url = sanitize_text_field($_POST['url'] ?? '');
	86	+
	87	+ if (empty($process_type) || empty($url)) {
	88	+ wp_send_json_error('Missing required parameters');
	89	+ }
	90	+
	91	+ $processed = 0;
	92	+
	93	+ if ($process_type === 'pdf') {
	94	+ $processed = $this->mxchat_manual_process_pdf_batch($url);
	95	+ } elseif ($process_type === 'sitemap') {
	96	+ $processed = $this->mxchat_manual_process_sitemap_batch($url);
	97	+ }
	98	+
	99	+ if ($processed > 0) {
	100	+ wp_send_json_success(array(
	101	+ 'message' => "Processed {$processed} items successfully",
	102	+ 'processed' => $processed
	103	+ ));
	104	+ } else {
	105	+ wp_send_json_error('No items were processed');
	106	+ }
	107	+
	108	+ } catch (Exception $e) {
	109	+ //error_log('Manual batch process error: ' . $e->getMessage());
	110	+ wp_send_json_error('Processing failed: ' . $e->getMessage());
	111	+ }
	112	+}
126	113
	114	+/**
	115	+ * Process a small PDF batch manually - DIRECT PROCESSING
	116	+ */
	117	+private function mxchat_manual_process_pdf_batch($pdf_url) {
	118	+ try {
	119	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	120	+ $status = get_transient($status_key);
	121	+
	122	+ if (!$status || $status['status'] !== 'processing') {
	123	+ //error_log('Manual PDF: No processing status found');
	124	+ return 0;
	125	+ }
	126	+
	127	+ //error_log('Manual PDF: Starting direct processing for ' . $pdf_url);
	128	+
	129	+ // Get current progress
	130	+ $current_page = $status['processed_pages'] ?? 0;
	131	+ $total_pages = $status['total_pages'] ?? 0;
	132	+
	133	+ if ($current_page >= $total_pages) {
	134	+ //error_log('Manual PDF: Already completed');
	135	+ return 0;
	136	+ }
	137	+
	138	+ // Try to download the PDF again for processing
	139	+ $response = wp_remote_get($pdf_url, array('timeout' => 30));
	140	+
	141	+ if (is_wp_error($response)) {
	142	+ //error_log('Manual PDF: Failed to download PDF: ' . $response->get_error_message());
	143	+ return 0;
	144	+ }
	145	+
	146	+ $pdf_content = wp_remote_retrieve_body($response);
	147	+ if (empty($pdf_content)) {
	148	+ //error_log('Manual PDF: Empty PDF content');
	149	+ return 0;
	150	+ }
	151	+
	152	+ // Save PDF temporarily
	153	+ $upload_dir = wp_upload_dir();
	154	+ $temp_pdf_path = trailingslashit($upload_dir['path']) . 'temp_manual_' . time() . '.pdf';
	155	+ file_put_contents($temp_pdf_path, $pdf_content);
	156	+
	157	+ // Process 2 pages directly
	158	+ $processed = $this->mxchat_process_pdf_pages_direct($temp_pdf_path, $pdf_url, $current_page, 5);
	159	+
	160	+ // Clean up temp file
	161	+ if (file_exists($temp_pdf_path)) {
	162	+ wp_delete_file($temp_pdf_path);
	163	+ }
	164	+
	165	+ //error_log('Manual PDF: Processed ' . $processed . ' pages');
	166	+ return $processed;
	167	+
	168	+ } catch (Exception $e) {
	169	+ //error_log('Manual PDF batch error: ' . $e->getMessage());
	170	+ return 0;
	171	+ }
	172	+}
	173	+
	174	+/**
	175	+ * Process PDF pages directly without cron
	176	+ */
	177	+private function mxchat_process_pdf_pages_direct($pdf_path, $pdf_url, $start_page, $batch_size) {
	178	+ try {
	179	+ if (!file_exists($pdf_path)) {
	180	+ //error_log('Direct PDF: File not found at ' . $pdf_path);
	181	+ return 0;
	182	+ }
	183	+
	184	+ $parser = new \Smalot\PdfParser\Parser();
	185	+ $pdf = $parser->parseFile($pdf_path);
	186	+ $pages = $pdf->getPages();
	187	+
	188	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	189	+ $status = get_transient($status_key);
	190	+
	191	+ if (!$status) {
	192	+ return 0;
	193	+ }
	194	+
	195	+ $options = get_option('mxchat_options');
	196	+ $api_key = $options['api_key'] ?? '';
	197	+
	198	+ if (empty($api_key)) {
	199	+ //error_log('Direct PDF: No API key');
	200	+ return 0;
	201	+ }
	202	+
	203	+ $processed = 0;
	204	+ $end_page = min($start_page + $batch_size, count($pages));
	205	+
	206	+ for ($i = $start_page; $i < $end_page; $i++) {
	207	+ try {
	208	+ $page_number = $i + 1;
	209	+ $text = $pages[$i]->getText();
	210	+
	211	+ if (empty($text)) {
	212	+ //error_log('Direct PDF: Empty text on page ' . $page_number);
	213	+ continue;
	214	+ }
	215	+
	216	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
	217	+ if (empty($sanitized_content)) {
	218	+ //error_log('Direct PDF: No content after sanitization on page ' . $page_number);
	219	+ continue;
	220	+ }
	221	+
	222	+ // Generate embedding
	223	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	224	+ if (is_string($embedding_vector)) {
	225	+ //error_log('Direct PDF: Embedding failed on page ' . $page_number . ': ' . $embedding_vector);
	226	+ continue;
	227	+ }
	228	+
	229	+ // Create metadata
	230	+ $metadata = array(
	231	+ 'document_type' => 'pdf',
	232	+ 'total_pages' => count($pages),
	233	+ 'current_page' => $page_number,
	234	+ 'source_url' => $pdf_url
	235	+ );
	236	+
	237	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	238	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
	239	+
	240	+ // Store in database
	241	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $api_key);
	242	+
	243	+ if (is_wp_error($db_result)) {
	244	+ //error_log('Direct PDF: DB error on page ' . $page_number . ': ' . $db_result->get_error_message());
	245	+ continue;
	246	+ }
	247	+
	248	+ $processed++;
	249	+ //error_log('Direct PDF: Successfully processed page ' . $page_number);
	250	+
	251	+ // Update status
	252	+ $status['processed_pages'] = $i + 1;
	253	+ $status['last_update'] = time();
	254	+ $status['percentage'] = round(($status['processed_pages'] / $status['total_pages']) * 100);
	255	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	256	+
	257	+ } catch (Exception $e) {
	258	+ //error_log('Direct PDF: Error processing page ' . ($i + 1) . ': ' . $e->getMessage());
	259	+ continue;
	260	+ }
	261	+ }
	262	+
	263	+ // Check if completed
	264	+ if ($status['processed_pages'] >= $status['total_pages']) {
	265	+ $status['status'] = 'complete';
	266	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	267	+ //error_log('Direct PDF: Processing completed');
	268	+ }
	269	+
	270	+ return $processed;
	271	+
	272	+ } catch (Exception $e) {
	273	+ //error_log('Direct PDF processing error: ' . $e->getMessage());
	274	+ return 0;
	275	+ }
	276	+}
	277	+
	278	+/**
	279	+ * Process a small sitemap batch manually - DIRECT PROCESSING
	280	+ */
	281	+private function mxchat_manual_process_sitemap_batch($sitemap_url) {
	282	+ try {
	283	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	284	+ $status = get_transient($status_key);
	285	+
	286	+ if (!$status || $status['status'] !== 'processing') {
	287	+ return 0;
	288	+ }
	289	+
	290	+ //error_log('Manual Sitemap: Starting direct processing for ' . $sitemap_url);
	291	+
	292	+ // Re-fetch the sitemap to get URLs
	293	+ $response = wp_remote_get($sitemap_url, array('timeout' => 30));
	294	+ if (is_wp_error($response)) {
	295	+ //error_log('Manual Sitemap: Failed to fetch sitemap');
	296	+ return 0;
	297	+ }
	298	+
	299	+ $sitemap_content = wp_remote_retrieve_body($response);
	300	+ $xml = simplexml_load_string($sitemap_content);
	301	+
	302	+ if (!$xml) {
	303	+ //error_log('Manual Sitemap: Invalid XML');
	304	+ return 0;
	305	+ }
	306	+
	307	+ $urls = array();
	308	+ foreach ($xml->url as $url_element) {
	309	+ $urls[] = (string)$url_element->loc;
	310	+ }
	311	+
	312	+ $current_processed = $status['processed_urls'] ?? 0;
	313	+ $batch_size = 5;
	314	+ $processed = 0;
	315	+
	316	+ // Process next 2 URLs
	317	+ for ($i = $current_processed; $i < min($current_processed + $batch_size, count($urls)); $i++) {
	318	+ $url = $urls[$i];
	319	+
	320	+ if ($this->mxchat_process_single_url_direct($url)) {
	321	+ $processed++;
	322	+ }
	323	+
	324	+ // Update status
	325	+ $status['processed_urls'] = $i + 1;
	326	+ $status['last_update'] = time();
	327	+ $status['percentage'] = round(($status['processed_urls'] / $status['total_urls']) * 100);
	328	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	329	+ }
	330	+
	331	+ // Check if completed
	332	+ if ($status['processed_urls'] >= $status['total_urls']) {
	333	+ $status['status'] = 'complete';
	334	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	335	+ }
	336	+
	337	+ //error_log('Manual Sitemap: Processed ' . $processed . ' URLs');
	338	+ return $processed;
	339	+
	340	+ } catch (Exception $e) {
	341	+ //error_log('Manual sitemap batch error: ' . $e->getMessage());
	342	+ return 0;
	343	+ }
	344	+}
	345	+
	346	+/**
	347	+ * Process a single URL directly
	348	+ */
	349	+private function mxchat_process_single_url_direct($url) {
	350	+ try {
	351	+ $response = wp_remote_get($url, array('timeout' => 30));
	352	+ if (is_wp_error($response)) {
	353	+ return false;
	354	+ }
	355	+
	356	+ $html = wp_remote_retrieve_body($response);
	357	+ $content = $this->mxchat_extract_main_content($html);
	358	+ $sanitized = $this->mxchat_sanitize_content_for_api($content);
	359	+
	360	+ if (empty($sanitized)) {
	361	+ return false;
	362	+ }
	363	+
	364	+ $options = get_option('mxchat_options');
	365	+ $api_key = $options['api_key'] ?? '';
	366	+
	367	+ $result = MxChat_Utils::submit_content_to_db($sanitized, $url, $api_key);
	368	+
	369	+ return !is_wp_error($result);
	370	+
	371	+ } catch (Exception $e) {
	372	+ //error_log('Single URL processing error: ' . $e->getMessage());
	373	+ return false;
	374	+ }
	375	+}
	376	+
	377	+
127	378	// ========================================
128	379	// MAIN CONTENT SUBMISSION HANDLERS
129	380	// ========================================
130	381
		@@ -140,30 +391,34 @@
140	391	wp_die(esc_html__('Nonce verification failed.', 'mxchat'));
141	392	}
142	393
143	394	// Sanitize the inputs.
144		- // Use wp_kses_post to allow safe HTML and wp_unslash to remove WordPress added slashes
145		- $article_content = wp_kses_post(wp_unslash($_POST['article_content']));
	395	+ $article_content = sanitize_textarea_field($_POST['article_content']);
146	396	$article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
147	397
148		- // Get bot_id from form submission
149		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
	398	+ // Get API key for submission
	399	+ $options = get_option('mxchat_options');
	400	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
150	401
151		- // Get bot-specific options and API key
152		- $bot_options = $this->get_bot_options($bot_id);
153		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
154		-
155		- // Custom-provider-aware decision; keyless custom sites must pass (plan cbd5fd).
156		- $preflight = MxChat_Utils::embedding_preflight($options);
157		- if (!$preflight['ok']) {
158		- set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
	402	+ if (strpos($selected_model, 'voyage') === 0) {
	403	+ $api_key = $options['voyage_api_key'] ?? '';
	404	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	405	+ $api_key = $options['gemini_api_key'] ?? '';
	406	+ } else {
	407	+ $api_key = $options['api_key'] ?? '';
	408	+ }
	409	+
	410	+ if (empty($api_key)) {
	411	+ set_transient('mxchat_admin_notice_error',
	412	+ esc_html__('API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
	413	+ 30
	414	+ );
159	415	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
160	416	exit;
161	417	}
162		- $api_key = $preflight['api_key'];
163	418
164		- // Use centralized utility function with bot_id
165		- $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key, null, $bot_id);
	419	+ // Use centralized utility function for storage
	420	+ $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key);
166	421
167	422	if (is_wp_error($result)) {
168	423	set_transient('mxchat_admin_notice_error',
169	424	esc_html__('Error storing content: ', 'mxchat') . $result->get_error_message(),
		@@ -178,253 +433,17 @@
178	433
179	434	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
180	435	exit;
181	436	}
182		-
183		-/**
184		- * Handle the "YouTube" KB import source (admin-post form submission).
185		- *
186		- * Per-video description mode:
187		- * - auto: fetch oEmbed metadata (reliable) + best-effort captions transcript.
188		- * If no usable transcript, index the metadata anyway, tell the admin,
189		- * and bounce back with the manual box pre-filled (never fail silently).
190		- * - manual: the admin's own description is what gets indexed; metadata rides along.
191		- *
192		- * The row is stored with content_type 'youtube' and source_url = the canonical
193		- * watch URL, so re-importing the same video UPDATES the entry (source_url
194		- * duplicate handling in MxChat_Utils::store_in_wordpress_db) — that is also the
195		- * "augment a metadata-only entry" path.
196		- */
197		-public function mxchat_handle_youtube_submission() {
198		- if (!isset($_POST['submit_youtube']) || !current_user_can('manage_options')) {
199		- wp_die(esc_html__('Unauthorized access', 'mxchat'));
200		- }
201		-
202		- check_admin_referer('mxchat_submit_youtube_action', 'mxchat_submit_youtube_nonce');
203		-
204		- $redirect_url = admin_url('admin.php?page=mxchat-prompts');
205		-
206		- $youtube_url = isset($_POST['youtube_url']) ? esc_url_raw(wp_unslash($_POST['youtube_url'])) : '';
207		- $video_id = MxChat_Utils::parse_youtube_id($youtube_url);
208		-
209		- if (empty($video_id)) {
210		- set_transient('mxchat_admin_notice_error',
211		- esc_html__('That does not look like a link to a single YouTube video. Please paste a watch, youtu.be, or Shorts URL.', 'mxchat'),
212		- 30
213		- );
214		- wp_safe_redirect(esc_url($redirect_url));
215		- exit;
216		- }
217		-
218		- $canonical_url = 'https://www.youtube.com/watch?v=' . $video_id;
219		-
220		- $description_mode = (isset($_POST['youtube_description_mode']) && $_POST['youtube_description_mode'] === 'manual') ? 'manual' : 'auto';
221		- $manual_description = isset($_POST['youtube_description']) ? trim(wp_kses_post(wp_unslash($_POST['youtube_description']))) : '';
222		-
223		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
224		-
225		- // Resolve the embedding decision exactly like the sibling handlers —
226		- // custom-provider-aware (plan cbd5fd).
227		- $bot_options = $this->get_bot_options($bot_id);
228		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
229		-
230		- $preflight = MxChat_Utils::embedding_preflight($options);
231		- if (!$preflight['ok']) {
232		- set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
233		- wp_safe_redirect(esc_url($redirect_url));
234		- exit;
235		- }
236		- $api_key = $preflight['api_key'];
237		-
238		- // Metadata is fetched in BOTH modes — it is the reliable half of auto, and in
239		- // manual mode it enriches the indexed text with the real title/channel.
240		- $meta = $this->mxchat_fetch_youtube_oembed($video_id);
241		- $video_title = isset($meta['title']) ? sanitize_text_field($meta['title']) : '';
242		- $video_channel = isset($meta['author_name']) ? sanitize_text_field($meta['author_name']) : '';
243		-
244		- $header_lines = 'YouTube Video: ' . ($video_title !== '' ? $video_title : $canonical_url) . "\n";
245		- if ($video_channel !== '') {
246		- $header_lines .= 'Channel: ' . $video_channel . "\n";
247		- }
248		- $header_lines .= 'URL: ' . $canonical_url . "\n\n";
249		-
250		- $transcript_missing = false;
251		-
252		- if ($description_mode === 'manual') {
253		- if ($manual_description === '') {
254		- set_transient('mxchat_admin_notice_error',
255		- esc_html__('Please write a description for the video, or switch to auto-fetch.', 'mxchat'),
256		- 30
257		- );
258		- wp_safe_redirect(esc_url($redirect_url));
259		- exit;
260		- }
261		- $indexed_text = $header_lines . $manual_description;
262		- } else {
263		- $transcript = $this->mxchat_fetch_youtube_transcript($video_id);
264		-
265		- if (strlen($transcript) >= 200) {
266		- $indexed_text = $header_lines . $transcript;
267		- } else {
268		- // Graceful fallback: captions disabled / blocked / no speech. Auto
269		- // reliably gets metadata; it does NOT guarantee a transcript.
270		- $transcript_missing = true;
271		-
272		- if ($video_title === '' && $video_channel === '') {
273		- // Both halves failed — nothing meaningful to index.
274		- set_transient('mxchat_admin_notice_error',
275		- esc_html__('Could not retrieve any information for that video (no metadata and no captions). Please check the URL, or use the manual description option.', 'mxchat'),
276		- 30
277		- );
278		- wp_safe_redirect(esc_url($redirect_url));
279		- exit;
280		- }
281		-
282		- $indexed_text = $header_lines . sprintf(
283		- /* translators: 1: video title, 2: channel name */
284		- __('A YouTube video titled "%1$s" from the channel %2$s.', 'mxchat'),
285		- $video_title !== '' ? $video_title : $canonical_url,
286		- $video_channel !== '' ? $video_channel : 'YouTube'
287		- );
288		- }
289		- }
290		-
291		- $result = MxChat_Utils::submit_content_to_db($indexed_text, $canonical_url, $api_key, null, $bot_id, 'youtube');
292		-
293		- if (is_wp_error($result)) {
294		- set_transient('mxchat_admin_notice_error',
295		- esc_html__('Error storing video in the knowledge base: ', 'mxchat') . $result->get_error_message(),
296		- 30
297		- );
298		- wp_safe_redirect(esc_url($redirect_url));
299		- exit;
300		- }
301		-
302		- if ($transcript_missing) {
303		- set_transient('mxchat_admin_notice_success',
304		- esc_html__('Video indexed from its title and channel — no captions were available for a transcript. The form below is pre-filled: write your own description and import again to improve matching (it updates the same entry).', 'mxchat'),
305		- 30
306		- );
307		- // Bounce back with prefill args so the page reopens the YouTube form in
308		- // manual mode with the URL + fetched title ready to augment.
309		- $redirect_url = add_query_arg(array(
310		- 'mxchat_yt_prefill' => '1',
311		- 'yt_url' => rawurlencode($canonical_url),
312		- 'yt_title' => rawurlencode($video_title),
313		- ), $redirect_url);
314		- } else {
315		- set_transient('mxchat_admin_notice_success',
316		- esc_html__('YouTube video successfully added to the knowledge base!', 'mxchat'),
317		- 30
318		- );
319		- }
320		-
321		- wp_safe_redirect(esc_url_raw($redirect_url));
322		- exit;
323		-}
324		-
325		-/**
326		- * Fetch YouTube oEmbed metadata for a video (no API key required).
327		- * Returns the decoded array (title, author_name, thumbnail_url, ...) or array().
328		- */
329		-private function mxchat_fetch_youtube_oembed($video_id) {
330		- $oembed_url = 'https://www.youtube.com/oembed?url=' . rawurlencode('https://www.youtube.com/watch?v=' . $video_id) . '&format=json';
331		- $response = wp_remote_get($oembed_url, array('timeout' => 15));
332		- if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
333		- return array();
334		- }
335		- $data = json_decode(wp_remote_retrieve_body($response), true);
336		- return is_array($data) ? $data : array();
337		-}
338		-
339		-/**
340		- * Best-effort captions transcript for a video. Deliberately ISOLATED: this uses
341		- * YouTube's unofficial timedtext route (the caption track list embedded in the
342		- * watch page), which YouTube has broken before and will break again. Every
343		- * failure mode returns '' so a break degrades to the metadata-only import path
344		- * instead of erroring the whole submission. Do not let anything in here throw.
345		- */
346		-private function mxchat_fetch_youtube_transcript($video_id) {
347		- $watch_url = 'https://www.youtube.com/watch?v=' . $video_id . '&hl=en';
348		-
349		- // First try the honest ingest UA; some responses omit the player config for
350		- // bot UAs, so retry once with a browser UA before giving up.
351		- $user_agents = array(
352		- mxchat_ingest_user_agent(),
353		- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
354		- );
355		-
356		- $tracks = array();
357		- foreach ($user_agents as $ua) {
358		- $response = wp_remote_get($watch_url, array(
359		- 'timeout' => 20,
360		- 'user-agent' => $ua,
361		- ));
362		- if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
363		- continue;
364		- }
365		- $body = wp_remote_retrieve_body($response);
366		- if (!is_string($body) || $body === '' || !preg_match('/"captionTracks":(\[.*?\])(?=,")/s', $body, $m)) {
367		- continue;
368		- }
369		- $decoded = json_decode($m[1], true);
370		- if (is_array($decoded) && !empty($decoded)) {
371		- $tracks = $decoded;
372		- break;
373		- }
374		- }
375		-
376		- if (empty($tracks)) {
377		- return '';
378		- }
379		-
380		- // Prefer an English track, else take the first offered.
381		- $chosen = null;
382		- foreach ($tracks as $track) {
383		- if (isset($track['languageCode']) && strpos($track['languageCode'], 'en') === 0) {
384		- $chosen = $track;
385		- break;
386		- }
387		- }
388		- if ($chosen === null) {
389		- $chosen = $tracks[0];
390		- }
391		- if (empty($chosen['baseUrl']) || !is_string($chosen['baseUrl'])) {
392		- return '';
393		- }
394		-
395		- $timedtext = wp_remote_get($chosen['baseUrl'], array('timeout' => 20));
396		- if (is_wp_error($timedtext) || wp_remote_retrieve_response_code($timedtext) !== 200) {
397		- return '';
398		- }
399		- $xml = wp_remote_retrieve_body($timedtext);
400		- if (!is_string($xml) || strpos($xml, '<text') === false) {
401		- return '';
402		- }
403		-
404		- // <text start=".." dur="..">caption</text> — strip tags, decode the
405		- // double-encoded entities timedtext ships, collapse whitespace.
406		- $text = preg_replace('/<[^>]+>/', ' ', $xml);
407		- $text = html_entity_decode(html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8'), ENT_QUOTES | ENT_HTML5, 'UTF-8');
408		- $text = trim(preg_replace('/\s+/u', ' ', $text));
409		-
410		- return $text;
411		-}
412		-
413	437	public function mxchat_is_pdf_url($url, $response) {
414	438	$content_type = wp_remote_retrieve_header($response, 'content-type');
415	439	$file_extension = strtolower(pathinfo($url, PATHINFO_EXTENSION));
416	440
417		- // Check Content-Disposition header for .pdf filename (Google Drive sends this)
418		- $disposition = wp_remote_retrieve_header($response, 'content-disposition');
419		- $has_pdf_disposition = ! empty($disposition) && stripos($disposition, '.pdf') !== false;
420		-
421		- return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf' || $has_pdf_disposition;
	441	+ return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf';
422	442	}
423		-
424		-
425		-public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response, $bot_id = 'default') {
	443	+public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response) {
426	444	if (!current_user_can('manage_options')) {
	445	+ //error_log(esc_html__('Unauthorized PDF processing attempt', 'mxchat'));
427	446	return false;
428	447	}
429	448
430	449	$pdf_url = esc_url_raw($pdf_url);
		@@ -430,8 +449,9 @@
430	449	$pdf_url = esc_url_raw($pdf_url);
431	450	$upload_dir = wp_upload_dir();
432	451
433	452	if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
	453	+ //error_log(sprintf(esc_html__('Upload directory error: %s', 'mxchat'), esc_html($upload_dir['error'])));
434	454	return false;
435	455	}
436	456
437	457	$pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
		@@ -438,12 +458,14 @@
438	458	$pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
439	459
440	460	$response_body = wp_remote_retrieve_body($response);
441	461	if (empty($response_body)) {
	462	+ //error_log(esc_html__('Empty PDF response body', 'mxchat'));
442	463	return false;
443	464	}
444	465
445	466	if (!wp_mkdir_p(dirname($pdf_path))) {
	467	+ //error_log(sprintf(esc_html__('Failed to create directory for PDF: %s', 'mxchat'), esc_html($pdf_path)));
446	468	return false;
447	469	}
448	470
449	471	try {
		@@ -452,925 +474,295 @@
452	474	if (!file_exists($pdf_path)) {
453	475	throw new Exception(__('Failed to save PDF file', 'mxchat'));
454	476	}
455	477
456		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
457		-
458		- if ($total_pages === false || $total_pages < 1) {
459		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
460		- }
	478	+ $parser = new \Smalot\PdfParser\Parser();
	479	+ $pdf = $parser->parseFile($pdf_path);
	480	+ $total_pages = absint(count($pdf->getPages()));
461	481
462		- // Create unique queue ID
463		- $queue_id = 'pdf_' . md5($pdf_url . time());
464		-
465		- // Create array of pages to process
466		- $pages = array();
467		- for ($i = 1; $i <= $total_pages; $i++) {
468		- $pages[] = array(
469		- 'pdf_path' => $pdf_path,
470		- 'pdf_url' => $pdf_url,
471		- 'page_number' => $i,
472		- 'total_pages' => $total_pages
473		- );
	482	+ if ($total_pages < 1) {
	483	+ throw new Exception(__('Invalid PDF: no pages found', 'mxchat'));
474	484	}
475	485
476		- // Add pages to queue
477		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
	486	+ wp_schedule_single_event(time(), 'mxchat_process_pdf_pages', array(
	487	+ 'pdf_path' => $pdf_path,
	488	+ 'pdf_url' => $pdf_url,
	489	+ 'total_pages' => $total_pages,
	490	+ 'batch_size' => absint(15),
	491	+ 'batch_pause' => absint(10)
	492	+ ));
478	493
479		- if ($queued_count === 0) {
480		- wp_delete_file($pdf_path);
481		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
482		- }
	494	+ $status_data = array(
	495	+ 'total_pages' => $total_pages,
	496	+ 'processed_pages' => 0,
	497	+ 'status' => 'processing',
	498	+ 'last_update' => time()
	499	+ );
483	500
484		- // Store queue metadata
485		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $pdf_url);
486		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
487		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
488		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
489		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
490		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	501	+ set_transient(
	502	+ sanitize_key('mxchat_pdf_status_' . md5($pdf_url)),
	503	+ array_map('sanitize_text_field', $status_data),
	504	+ DAY_IN_SECONDS
	505	+ );
491	506
492		- // Store queue ID in transient for status tracking
493		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
494		- set_transient('mxchat_last_pdf_url', $pdf_url, DAY_IN_SECONDS);
	507	+ return __('scheduled', 'mxchat');
495	508
496		- return 'queued';
497		-
498	509	} catch (Exception $e) {
	510	+ //error_log(sprintf(esc_html__('Error preparing PDF for processing: %s', 'mxchat'), esc_html($e->getMessage())));
499	511	if (file_exists($pdf_path)) {
500	512	wp_delete_file($pdf_path);
501	513	}
502		- return $e->getMessage();
	514	+ return false;
503	515	}
504	516	}
505	517
506		-/**
507		- * Handle direct PDF file upload from the knowledge base page
508		- */
509		-public function mxchat_handle_pdf_file_submission() {
510		- if (!isset($_POST['submit_pdf_file']) || !current_user_can('manage_options')) {
511		- wp_die(esc_html__('Unauthorized access', 'mxchat'));
512		- }
	518	+public function mxchat_process_pdf_pages_cron($pdf_path, $pdf_url, $total_pages, $batch_size, $batch_pause) {
	519	+ // Validate inputs
	520	+ $pdf_path = sanitize_text_field($pdf_path);
	521	+ $pdf_url = esc_url_raw($pdf_url);
	522	+ $total_pages = absint($total_pages);
	523	+ $batch_size = absint($batch_size);
	524	+ $batch_pause = absint($batch_pause);
513	525
514		- check_admin_referer('mxchat_submit_pdf_file_action', 'mxchat_submit_pdf_file_nonce');
515		-
516		- $redirect_url = admin_url('admin.php?page=mxchat-prompts');
517		-
518		- // Validate file upload
519		- if (empty($_FILES['pdf_file']) || $_FILES['pdf_file']['error'] !== UPLOAD_ERR_OK) {
520		- $error_code = isset($_FILES['pdf_file']['error']) ? $_FILES['pdf_file']['error'] : UPLOAD_ERR_NO_FILE;
521		- $error_messages = array(
522		- UPLOAD_ERR_INI_SIZE => __('The uploaded file exceeds the server upload_max_filesize limit.', 'mxchat'),
523		- UPLOAD_ERR_FORM_SIZE => __('The uploaded file exceeds the form MAX_FILE_SIZE limit.', 'mxchat'),
524		- UPLOAD_ERR_PARTIAL => __('The file was only partially uploaded.', 'mxchat'),
525		- UPLOAD_ERR_NO_FILE => __('No file was uploaded. Please select a PDF file.', 'mxchat'),
526		- UPLOAD_ERR_NO_TMP_DIR => __('Server missing temporary folder.', 'mxchat'),
527		- UPLOAD_ERR_CANT_WRITE => __('Server failed to write file to disk.', 'mxchat'),
528		- );
529		- $error_msg = isset($error_messages[$error_code]) ? $error_messages[$error_code] : __('Unknown upload error.', 'mxchat');
530		- set_transient('mxchat_admin_notice_error', $error_msg, 30);
531		- wp_safe_redirect(esc_url($redirect_url));
532		- exit;
533		- }
534		-
535		- $file = $_FILES['pdf_file'];
536		-
537		- // Validate MIME type
538		- $finfo = finfo_open(FILEINFO_MIME_TYPE);
539		- $mime_type = finfo_file($finfo, $file['tmp_name']);
540		- finfo_close($finfo);
541		-
542		- if ($mime_type !== 'application/pdf') {
543		- set_transient('mxchat_admin_notice_error',
544		- esc_html__('Invalid file type. Only PDF files are accepted.', 'mxchat'),
545		- 30
546		- );
547		- wp_safe_redirect(esc_url($redirect_url));
548		- exit;
549		- }
550		-
551		- // Validate extension
552		- $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
553		- if ($ext !== 'pdf') {
554		- set_transient('mxchat_admin_notice_error',
555		- esc_html__('Invalid file extension. Only .pdf files are accepted.', 'mxchat'),
556		- 30
557		- );
558		- wp_safe_redirect(esc_url($redirect_url));
559		- exit;
560		- }
561		-
562		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
563		- $original_filename = sanitize_file_name($file['name']);
564		-
565		- $upload_dir = wp_upload_dir();
566		- if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
567		- set_transient('mxchat_admin_notice_error',
568		- esc_html__('WordPress upload directory is not writable.', 'mxchat'),
569		- 30
570		- );
571		- wp_safe_redirect(esc_url($redirect_url));
572		- exit;
573		- }
574		-
575		- $pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
576		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
577		-
578		- if (!wp_mkdir_p(dirname($pdf_path))) {
579		- set_transient('mxchat_admin_notice_error',
580		- esc_html__('Failed to create upload directory.', 'mxchat'),
581		- 30
582		- );
583		- wp_safe_redirect(esc_url($redirect_url));
584		- exit;
585		- }
586		-
587		- // Move uploaded file
588		- if (!move_uploaded_file($file['tmp_name'], $pdf_path)) {
589		- set_transient('mxchat_admin_notice_error',
590		- esc_html__('Failed to save uploaded PDF file.', 'mxchat'),
591		- 30
592		- );
593		- wp_safe_redirect(esc_url($redirect_url));
594		- exit;
595		- }
596		-
597	526	try {
598		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
599		-
600		- if ($total_pages === false || $total_pages < 1) {
601		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
	527	+ if (!file_exists($pdf_path)) {
	528	+ throw new Exception(sprintf('PDF file not found at path: %s', esc_html($pdf_path)));
602	529	}
603	530
604		- // Use original filename as the source identifier
605		- $source_label = 'upload://' . $original_filename;
606		-
607		- $queue_id = 'pdf_' . md5($source_label . time());
608		-
609		- $pages = array();
610		- for ($i = 1; $i <= $total_pages; $i++) {
611		- $pages[] = array(
612		- 'pdf_path' => $pdf_path,
613		- 'pdf_url' => $source_label,
614		- 'page_number' => $i,
615		- 'total_pages' => $total_pages,
616		- );
617		- }
618		-
619		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
620		-
621		- if ($queued_count === 0) {
622		- wp_delete_file($pdf_path);
623		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
624		- }
625		-
626		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $source_label);
627		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
628		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
629		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
630		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
631		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
632		-
633		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
634		- set_transient('mxchat_last_pdf_url', $source_label, DAY_IN_SECONDS);
635		-
636		- set_transient('mxchat_admin_notice_success',
637		- sprintf(
638		- esc_html__('PDF "%s" (%d pages) queued for processing. Processing will start automatically.', 'mxchat'),
639		- esc_html($original_filename),
640		- $total_pages
641		- ),
642		- 30
643		- );
644		-
645		- } catch (Exception $e) {
646		- if (file_exists($pdf_path)) {
647		- wp_delete_file($pdf_path);
648		- }
649		- set_transient('mxchat_admin_notice_error',
650		- esc_html__('Failed to process uploaded PDF: ', 'mxchat') . esc_html($e->getMessage()),
651		- 30
652		- );
653		- }
654		-
655		- wp_safe_redirect(esc_url($redirect_url));
656		- exit;
657		-}
658		-
659		-/**
660		- * Validate PDF and count pages with multiple parser attempts
661		- */
662		-private function mxchat_validate_and_count_pdf_pages($pdf_path) {
663		- // Method 1: Try with Smalot PDF Parser (your current method)
664		- try {
665		- mxchat_load_pdf_parser();
666	531	$parser = new \Smalot\PdfParser\Parser();
667	532	$pdf = $parser->parseFile($pdf_path);
668	533	$pages = $pdf->getPages();
669		- $page_count = count($pages);
670		-
671		- if ($page_count > 0) {
672		- //error_log('PDF parsed successfully with Smalot parser: ' . $page_count . ' pages');
673		- return $page_count;
674		- }
675		- } catch (Exception $e) {
676		- //error_log('Smalot PDF parser failed: ' . $e->getMessage());
677		- }
678	534
679		- // Method 2: Try with pdfinfo command (if available)
680		- if (function_exists('shell_exec') && !$this->mxchat_is_shell_disabled()) {
681		- try {
682		- $command = 'pdfinfo ' . escapeshellarg($pdf_path) . ' 2>&1';
683		- $output = shell_exec($command);
684		-
685		- if ($output && preg_match('/Pages:\s*(\d+)/', $output, $matches)) {
686		- $page_count = intval($matches[1]);
687		- if ($page_count > 0) {
688		- //error_log('PDF parsed successfully with pdfinfo: ' . $page_count . ' pages');
689		- return $page_count;
690		- }
691		- }
692		- } catch (Exception $e) {
693		- //error_log('pdfinfo command failed: ' . $e->getMessage());
694		- }
695		- }
	535	+ // Get current progress
	536	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	537	+ $status = get_transient($status_key);
696	538
697		- // Method 3: Try to repair PDF and parse again
698		- try {
699		- $repaired_path = $this->mxchat_attempt_pdf_repair($pdf_path);
700		- if ($repaired_path && $repaired_path !== $pdf_path) {
701		- mxchat_load_pdf_parser();
702		- $parser = new \Smalot\PdfParser\Parser();
703		- $pdf = $parser->parseFile($repaired_path);
704		- $pages = $pdf->getPages();
705		- $page_count = count($pages);
706		-
707		- if ($page_count > 0) {
708		- // Replace original with repaired version
709		- copy($repaired_path, $pdf_path);
710		- unlink($repaired_path);
711		- //error_log('PDF repaired and parsed successfully: ' . $page_count . ' pages');
712		- return $page_count;
713		- }
714		-
715		- // Clean up repaired file if it didn't work
716		- unlink($repaired_path);
	539	+ if (!$status || !is_array($status)) {
	540	+ throw new Exception('Invalid status data retrieved from transient');
717	541	}
718		- } catch (Exception $e) {
719		- //error_log('PDF repair attempt failed: ' . $e->getMessage());
720		- }
721	542
722		- // Method 4: Manual PDF structure analysis (basic page count)
723		- try {
724		- $page_count = $this->mxchat_manual_pdf_page_count($pdf_path);
725		- if ($page_count > 0) {
726		- //error_log('PDF page count determined manually: ' . $page_count . ' pages');
727		- return $page_count;
	543	+ // Initialize failed pages list if it doesn't exist
	544	+ if (!isset($status['failed_pages_list']) || !is_array($status['failed_pages_list'])) {
	545	+ $status['failed_pages_list'] = [];
728	546	}
729		- } catch (Exception $e) {
730		- //error_log('Manual PDF analysis failed: ' . $e->getMessage());
731		- }
732	547
733		- //error_log('All PDF parsing methods failed for: ' . $pdf_path);
734		- return false;
735		-}
736		-
737		-/**
738		- * Check if shell_exec is disabled
739		- */
740		-private function mxchat_is_shell_disabled() {
741		- $disabled = explode(',', ini_get('disable_functions'));
742		- return in_array('shell_exec', $disabled);
743		-}
744		-
745		-/**
746		- * Attempt to repair PDF using basic methods
747		- */
748		-private function mxchat_attempt_pdf_repair($pdf_path) {
749		- try {
750		- $content = file_get_contents($pdf_path);
751		- if (!$content) {
752		- return false;
	548	+ $start_page = absint($status['processed_pages']);
	549	+ $end_page = min($start_page + $batch_size, $total_pages);
	550	+ $options = get_option('mxchat_options');
	551	+
	552	+ if (empty($options['api_key'])) {
	553	+ throw new Exception('API key is missing or invalid');
753	554	}
754	555
755		- // Check if PDF starts with proper header
756		- if (substr($content, 0, 4) !== '%PDF') {
757		- // Try to find PDF header in the content
758		- $header_pos = strpos($content, '%PDF');
759		- if ($header_pos !== false && $header_pos < 1024) {
760		- // Remove junk before PDF header
761		- $content = substr($content, $header_pos);
762		- $repaired_path = $pdf_path . '.repaired';
763		- file_put_contents($repaired_path, $content);
764		- return $repaired_path;
765		- }
766		- }
	556	+ $successful_pages = 0;
	557	+ $failed_pages = 0;
767	558
768		- // Check for EOF marker
769		- $content = rtrim($content);
770		- if (!preg_match('/%%EOF\s*$/', $content)) {
771		- // Add EOF marker if missing
772		- $content .= "\n%%EOF";
773		- $repaired_path = $pdf_path . '.repaired';
774		- file_put_contents($repaired_path, $content);
775		- return $repaired_path;
776		- }
	559	+ for ($i = $start_page; $i < $end_page; $i++) {
	560	+ $page_number = $i + 1;
	561	+ $max_retries = 3;
	562	+ $retry_count = 0;
	563	+ $page_processed = false;
	564	+ $last_error = '';
777	565
778		- } catch (Exception $e) {
779		- //error_log('PDF repair error: ' . $e->getMessage());
780		- }
	566	+ while (!$page_processed && $retry_count < $max_retries) {
	567	+ try {
	568	+ $text = $pages[$i]->getText();
	569	+
	570	+ if (empty($text)) {
	571	+ throw new Exception("Empty text on page {$page_number}");
	572	+ }
	573	+
	574	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
781	575
782		- return false;
783		-}
	576	+ if (empty($sanitized_content)) {
	577	+ throw new Exception("No valid content after sanitization on page {$page_number}");
	578	+ }
784	579
785		-/**
786		- * Manual PDF page counting by analyzing PDF structure
787		- */
788		-private function mxchat_manual_pdf_page_count($pdf_path) {
789		- try {
790		- $content = file_get_contents($pdf_path);
791		- if (!$content) {
792		- return 0;
793		- }
	580	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	581	+
	582	+ if (is_string($embedding_vector)) {
	583	+ throw new Exception("Embedding generation failed: " . $embedding_vector);
	584	+ }
	585	+
	586	+ if (!is_array($embedding_vector)) {
	587	+ throw new Exception("Embedding generation returned unexpected result type: " . gettype($embedding_vector));
	588	+ }
	589	+
	590	+ $metadata = array(
	591	+ 'document_type' => 'pdf',
	592	+ 'total_pages' => $total_pages,
	593	+ 'current_page' => $page_number,
	594	+ 'prev_page' => $i > 0 ? $i : null,
	595	+ 'next_page' => $i < ($total_pages - 1) ? $i + 2 : null,
	596	+ 'source_url' => $pdf_url
	597	+ );
794	598
795		- // Method 1: Count /Type /Page objects
796		- $page_count = preg_match_all('/\/Type\s*\/Page[^s]/', $content);
797		- if ($page_count > 0) {
798		- return $page_count;
799		- }
	599	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	600	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
800	601
801		- // Method 2: Look for /Count in pages object
802		- if (preg_match('/\/Type\s*\/Pages.*?\/Count\s+(\d+)/', $content, $matches)) {
803		- return intval($matches[1]);
804		- }
	602	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $options['api_key']);
	603	+
	604	+ if (is_wp_error($db_result)) {
	605	+ throw new Exception("Database submission failed: " . $db_result->get_error_message());
	606	+ }
805	607
806		- // Method 3: Count page references
807		- $page_count = preg_match_all('/\d+\s+0\s+obj\s*<<[^>]*\/Type\s*\/Page/', $content);
808		- if ($page_count > 0) {
809		- return $page_count;
810		- }
811		-
812		- } catch (Exception $e) {
813		- //error_log('Manual PDF analysis error: ' . $e->getMessage());
814		- }
815		-
816		- return 0;
817		-}
818		-
819		-
820		-public function mxchat_save_inline_prompt() {
821		- // DEBUG: Log what we're receiving
822		- //error_log('=== MXCHAT DEBUG ===');
823		- //error_log('POST data: ' . print_r($_POST, true));
824		- //error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
825		-
826		- // Check for nonce security
827		- check_ajax_referer('mxchat_save_inline_nonce', '_ajax_nonce');
828		-
829		- // If we get here, nonce passed
830		- //error_log('Nonce verification PASSED');
831		-
832		- // Verify permissions
833		- if (!current_user_can('manage_options')) {
834		- wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
835		- return;
836		- }
837		-
838		- global $wpdb;
839		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
840		-
841		- // Validate and sanitize input data
842		- $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
843		- $article_content = isset($_POST['article_content']) ? wp_kses_post(wp_unslash($_POST['article_content'])) : '';
844		- $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
845		-
846		- if ($prompt_id > 0 && !empty($article_content)) {
847		- // Re-generate the embedding vector for the updated content
848		- $embedding_vector = $this->mxchat_generate_embedding($article_content);
849		- if (is_array($embedding_vector)) {
850		- // Serialize the embedding vector before storing it
851		- $embedding_vector_serialized = serialize($embedding_vector);
852		- // Update the prompt in the database
853		- $updated = $wpdb->update(
854		- $table_name,
855		- array(
856		- 'article_content' => $article_content,
857		- 'embedding_vector' => $embedding_vector_serialized,
858		- 'source_url' => $article_url,
859		- ),
860		- array('id' => $prompt_id),
861		- array('%s', '%s', '%s'),
862		- array('%d')
863		- );
864		- if ($updated !== false) {
865		- wp_send_json_success();
866		- } else {
867		- MxChat_Admin::mxchat_log_debug('knowledge_error', 'Database update failed when saving knowledge entry');
868		- wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
	608	+ // Success!
	609	+ $page_processed = true;
	610	+ $successful_pages++;
	611	+
	612	+ } catch (Exception $e) {
	613	+ $retry_count++;
	614	+ $last_error = $e->getMessage();
	615	+
	616	+ //error_log("PDF page {$page_number} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	617	+
	618	+ if ($retry_count < $max_retries) {
	619	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	620	+ sleep(pow(2, $retry_count - 1));
	621	+ }
	622	+ }
869	623	}
870		- } else {
871		- MxChat_Admin::mxchat_log_debug('embedding_error', 'Embedding generation failed for knowledge entry');
872		- wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
873		- }
874		- } else {
875		- wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
876		- }
877		-}
878	624
879		-
880		-/**
881		- * AJAX: Get full content for editing — reassembles chunks if needed.
882		- * Works for both WordPress DB and Pinecone entries.
883		- */
884		-public function ajax_mxchat_get_entry_content() {
885		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
886		-
887		- if ( ! current_user_can('manage_options') ) {
888		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
889		- }
890		-
891		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
892		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
893		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
894		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
895		-
896		- if ( $data_source === 'pinecone' ) {
897		- // Pinecone: fetch vectors by source_url, reassemble chunks
898		- $content = $this->get_pinecone_entry_content( $source_url, $entry_id, $bot_id );
899		- } else {
900		- // WordPress DB
901		- $content = $this->get_wordpress_entry_content( $source_url, $entry_id );
902		- }
903		-
904		- if ( is_wp_error( $content ) ) {
905		- wp_send_json_error( array( 'message' => $content->get_error_message() ) );
906		- }
907		-
908		- wp_send_json_success( $content );
909		-}
910		-
911		-/**
912		- * Get content from WordPress DB — reassembles chunks by source_url.
913		- */
914		-private function get_wordpress_entry_content( $source_url, $entry_id ) {
915		- global $wpdb;
916		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
917		-
918		- // If we have a source_url, check for chunks
919		- if ( ! empty( $source_url ) && strpos( $source_url, 'mxchat://' ) !== 0 ) {
920		- $rows = $wpdb->get_results( $wpdb->prepare(
921		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
922		- $source_url
923		- ) );
924		-
925		- if ( $rows && count( $rows ) > 1 ) {
926		- // Multiple rows = chunked. Reassemble.
927		- $chunks = array();
928		- foreach ( $rows as $row ) {
929		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
930		- $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
931		- $chunks[ $index ] = $parsed['text'];
	625	+ // If page still not processed after all retries, mark as failed
	626	+ if (!$page_processed) {
	627	+ $failed_pages++;
	628	+ $status['failed_pages_list'][] = [
	629	+ 'page' => $page_number,
	630	+ 'error' => $last_error,
	631	+ 'time' => time(),
	632	+ 'retries' => $max_retries
	633	+ ];
	634	+
	635	+ // Limit failed pages list to prevent memory issues
	636	+ if (count($status['failed_pages_list']) > 50) {
	637	+ $status['failed_pages_list'] = array_slice($status['failed_pages_list'], -50);
	638	+ }
	639	+
	640	+ //error_log("PDF page {$page_number} permanently failed after {$max_retries} attempts: " . $last_error);
932	641	}
933		- ksort( $chunks );
934		- return array(
935		- 'content' => implode( "\n\n", $chunks ),
936		- 'source_url' => $source_url,
937		- 'is_chunked' => true,
938		- 'chunk_count' => count( $chunks ),
939		- 'content_type' => $rows[0]->content_type,
940		- );
941		- } elseif ( $rows && count( $rows ) === 1 ) {
942		- $parsed = MxChat_Chunker::parse_stored_chunk( $rows[0]->article_content );
943		- return array(
944		- 'content' => $parsed['text'],
945		- 'source_url' => $source_url,
946		- 'entry_id' => $rows[0]->id,
947		- 'is_chunked' => false,
948		- 'content_type' => $rows[0]->content_type,
949		- );
950		- }
951		- }
952	642
953		- // Fallback: fetch by ID
954		- if ( $entry_id > 0 ) {
955		- $row = $wpdb->get_row( $wpdb->prepare(
956		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
957		- $entry_id
958		- ) );
959		- if ( $row ) {
960		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
961		- return array(
962		- 'content' => $parsed['text'],
963		- 'source_url' => $row->source_url,
964		- 'entry_id' => $row->id,
965		- 'is_chunked' => false,
966		- 'content_type' => $row->content_type,
967		- );
	643	+ // Update progress
	644	+ $status['processed_pages'] = absint($page_number);
	645	+ $status['last_update'] = time();
	646	+ $status['failed_pages'] = absint($status['failed_pages'] ?? 0) + ($page_processed ? 0 : 1);
	647	+
	648	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
968	649	}
969		- }
970	650
971		- return new WP_Error( 'not_found', 'Entry not found.' );
972		-}
973		-
974		-/**
975		- * Get content from Pinecone — fetches vectors by source_url, reassembles chunks.
976		- */
977		-private function get_pinecone_entry_content( $source_url, $entry_id, $bot_id ) {
978		- if ( ! class_exists('MxChat_Pinecone_Manager') ) {
979		- return new WP_Error( 'pinecone_unavailable', 'Pinecone manager not available.' );
980		- }
981		-
982		- // Get Pinecone config
983		- if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
984		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
985		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
986		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
987		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
988		- } else {
989		- $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
990		- $api_key = $bot_config['api_key'] ?? '';
991		- $host = $bot_config['host'] ?? '';
992		- $namespace = $bot_config['namespace'] ?? '';
993		- }
994		-
995		- if ( empty($host) || empty($api_key) ) {
996		- return new WP_Error( 'pinecone_config', 'Pinecone not configured.' );
997		- }
998		-
999		- // List vectors with the source_url prefix
1000		- $base_id = md5( $source_url );
1001		- $vector_ids = array( $base_id );
1002		-
1003		- // Find chunk vectors
1004		- // NOTE: Pinecone's /vectors/list is a GET endpoint with query parameters.
1005		- // A POST is answered 200-with-an-empty-body, which reads as "no vectors".
1006		- $list_url = "https://{$host}/vectors/list";
1007		- $list_params = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
1008		- if ( ! empty($namespace) ) {
1009		- $list_params['namespace'] = $namespace;
1010		- }
1011		-
1012		- $list_resp = wp_remote_get( $list_url . '?' . http_build_query( $list_params ), array(
1013		- 'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ),
1014		- 'timeout' => 15,
1015		- ) );
1016		-
1017		- if ( ! is_wp_error($list_resp) ) {
1018		- $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
1019		- if ( ! empty($list_data['vectors']) ) {
1020		- foreach ( $list_data['vectors'] as $v ) {
1021		- $vector_ids[] = $v['id'];
	651	+ // Schedule next batch if needed
	652	+ if ($end_page < $total_pages) {
	653	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_pdf_pages', array(
	654	+ 'pdf_path' => $pdf_path,
	655	+ 'pdf_url' => $pdf_url,
	656	+ 'total_pages' => $total_pages,
	657	+ 'batch_size' => $batch_size,
	658	+ 'batch_pause' => $batch_pause
	659	+ ));
	660	+ } else {
	661	+ // Processing complete
	662	+ $status['status'] = 'complete';
	663	+ $status['processed_pages'] = $total_pages;
	664	+
	665	+ // Add completion summary
	666	+ $status['completion_summary'] = [
	667	+ 'total_pages' => $total_pages,
	668	+ 'successful_pages' => $total_pages - absint($status['failed_pages'] ?? 0),
	669	+ 'failed_pages' => absint($status['failed_pages'] ?? 0),
	670	+ 'completion_time' => current_time('mysql')
	671	+ ];
	672	+
	673	+ // Save the completed status (don't delete it - let user dismiss manually)
	674	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	675	+
	676	+ // Clean up the temporary PDF file
	677	+ if (file_exists($pdf_path)) {
	678	+ wp_delete_file($pdf_path);
1022	679	}
	680	+
	681	+ // DON'T delete the status transients here - let user dismiss manually
1023	682	}
1024		- }
1025	683
1026		- // Fetch vectors with metadata
1027		- // NOTE: /vectors/fetch is a GET endpoint too, and Pinecone expects the ids
1028		- // repeated (ids=a&ids=b) — http_build_query would emit ids[0]=a, so build
1029		- // the query string explicitly.
1030		- $fetch_query = array();
1031		- foreach ( $vector_ids as $fetch_vid ) {
1032		- $fetch_query[] = 'ids=' . rawurlencode( $fetch_vid );
1033		- }
1034		- if ( ! empty($namespace) ) {
1035		- $fetch_query[] = 'namespace=' . rawurlencode( $namespace );
1036		- }
1037		-
1038		- $fetch_resp = wp_remote_get( "https://{$host}/vectors/fetch?" . implode( '&', $fetch_query ), array(
1039		- 'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ),
1040		- 'timeout' => 15,
1041		- ) );
1042		-
1043		- if ( is_wp_error($fetch_resp) ) {
1044		- return new WP_Error( 'pinecone_fetch', 'Failed to fetch from Pinecone.' );
1045		- }
1046		-
1047		- $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
1048		- $vectors = $fetch_data['vectors'] ?? array();
1049		-
1050		- if ( empty($vectors) ) {
1051		- return new WP_Error( 'not_found', 'Entry not found in Pinecone.' );
1052		- }
1053		-
1054		- // Reassemble chunks
1055		- $chunks = array();
1056		- $content_type = 'content';
1057		- foreach ( $vectors as $vid => $vector ) {
1058		- $meta = $vector['metadata'] ?? array();
1059		- $text = $meta['text'] ?? '';
1060		- $index = $meta['chunk_index'] ?? 0;
1061		- $content_type = $meta['type'] ?? 'content';
1062		- $chunks[ intval($index) ] = $text;
1063		- }
1064		- ksort( $chunks );
1065		-
1066		- return array(
1067		- 'content' => implode( "\n\n", $chunks ),
1068		- 'source_url' => $source_url,
1069		- 'is_chunked' => count($chunks) > 1,
1070		- 'chunk_count' => count($chunks),
1071		- 'content_type' => $content_type,
1072		- );
1073		-}
1074		-
1075		-/**
1076		- * AJAX: Inspect a knowledge entry — returns the per-chunk stored text + metadata
1077		- * WITHOUT collapsing it, so a site owner can see exactly what was indexed for an
1078		- * entry (plan-mxchat-20260628-d8cb4b). READ-ONLY: never re-embeds or mutates.
1079		- */
1080		-public function ajax_mxchat_inspect_entry() {
1081		- check_ajax_referer('mxchat_inspect_entry_nonce', 'nonce');
1082		-
1083		- if ( ! current_user_can('manage_options') ) {
1084		- wp_send_json_error( array( 'message' => esc_html__('Permission denied.', 'mxchat') ) );
1085		- }
1086		-
1087		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
1088		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
1089		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
1090		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1091		-
1092		- if ( $data_source === 'pinecone' ) {
1093		- $result = $this->inspect_pinecone_entry( $source_url, $entry_id, $bot_id );
1094		- } else {
1095		- $result = $this->inspect_wordpress_entry( $source_url, $entry_id );
1096		- }
1097		-
1098		- if ( is_wp_error( $result ) ) {
1099		- wp_send_json_error( array( 'message' => $result->get_error_message() ) );
1100		- }
1101		-
1102		- wp_send_json_success( $result );
1103		-}
1104		-
1105		-/**
1106		- * Read-only inspector for WordPress-DB entries. Mirrors get_wordpress_entry_content()
1107		- * but returns each STORED chunk's exact text + length (no implode), plus the assembled
1108		- * embedded text. This shows what is actually in the index, not a re-derivation from the post.
1109		- */
1110		-private function inspect_wordpress_entry( $source_url, $entry_id ) {
1111		- global $wpdb;
1112		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
1113		-
1114		- $rows = array();
1115		-
1116		- // Group by the real stored source_url — this INCLUDES "mxchat://" manual
1117		- // Direct Content entries (the spec's manual-entry case), which share one
1118		- // source_url across their chunk rows. Only the synthetic "_ungrouped_<id>"
1119		- // display key (invented by the table view for rows with no source_url) is
1120		- // excluded; those fall through to the entry_id lookup below.
1121		- if ( ! empty( $source_url ) && strpos( $source_url, '_ungrouped_' ) !== 0 ) {
1122		- $rows = $wpdb->get_results( $wpdb->prepare(
1123		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
1124		- $source_url
1125		- ) );
1126		- }
1127		-
1128		- // Fallback / manual "Direct Content" entries: fetch the single row by id.
1129		- if ( empty( $rows ) && $entry_id > 0 ) {
1130		- $row = $wpdb->get_row( $wpdb->prepare(
1131		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
1132		- $entry_id
1133		- ) );
1134		- if ( $row ) {
1135		- $rows = array( $row );
	684	+ } catch (\Exception $e) {
	685	+ //error_log(sprintf('[MXCHAT-PDF] Error processing PDF: %s', $e->getMessage()));
	686	+
	687	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	688	+ $status = get_transient($status_key);
	689	+
	690	+ if (!$status || !is_array($status)) {
	691	+ $status = array(
	692	+ 'total_pages' => $total_pages,
	693	+ 'processed_pages' => 0,
	694	+ 'status' => 'error',
	695	+ 'error' => sanitize_text_field($e->getMessage()),
	696	+ 'last_update' => time()
	697	+ );
	698	+ } else {
	699	+ $status['status'] = 'error';
	700	+ $status['error'] = sanitize_text_field($e->getMessage());
	701	+ $status['last_update'] = time();
1136	702	}
1137		- }
1138		-
1139		- if ( empty( $rows ) ) {
1140		- return new WP_Error( 'not_found', esc_html__('Entry not found in the local knowledge database.', 'mxchat') );
1141		- }
1142		-
1143		- $chunks = array();
1144		- $content_type = '';
1145		- foreach ( $rows as $row ) {
1146		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
1147		- $text = isset( $parsed['text'] ) ? $parsed['text'] : '';
1148		- $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
1149		- $content_type = $row->content_type;
1150		- $chunks[] = array(
1151		- 'index' => $index,
1152		- 'text' => $text,
1153		- 'length' => function_exists('mb_strlen') ? mb_strlen( $text ) : strlen( $text ),
1154		- 'row_id' => intval( $row->id ),
1155		- );
1156		- }
1157		-
1158		- usort( $chunks, function( $a, $b ) { return $a['index'] - $b['index']; } );
1159		-
1160		- $assembled = implode( "\n\n", wp_list_pluck( $chunks, 'text' ) );
1161		-
1162		- return array(
1163		- 'store' => 'wordpress',
1164		- 'source_url' => $source_url,
1165		- 'content_type' => $content_type,
1166		- 'is_chunked' => count( $chunks ) > 1,
1167		- 'chunk_count' => count( $chunks ),
1168		- 'assembled' => $assembled,
1169		- 'assembled_length' => function_exists('mb_strlen') ? mb_strlen( $assembled ) : strlen( $assembled ),
1170		- 'chunks' => array_values( $chunks ),
1171		- // WP-DB storage carries no separate vector metadata; surface that fact
1172		- // rather than letting the owner guess (the spec's taxonomy question).
1173		- 'metadata' => array(),
1174		- 'metadata_note' => esc_html__('Stored in the local WordPress database. Only the assembled text shown here is embedded — there are no separate vector metadata fields (e.g. taxonomy terms are not stored unless they were injected into the text itself).', 'mxchat'),
1175		- );
1176		-}
1177		-
1178		-/**
1179		- * Read-only inspector for Pinecone entries. Mirrors get_pinecone_entry_content()
1180		- * but keeps each vector's text + metadata instead of imploding, so the owner can
1181		- * confirm exactly which metadata fields (text/source_url/type/last_updated/created_at/bot_id)
1182		- * are present per chunk. READ-ONLY.
1183		- */
1184		-private function inspect_pinecone_entry( $source_url, $entry_id, $bot_id ) {
1185		- if ( ! class_exists('MxChat_Pinecone_Manager') ) {
1186		- return new WP_Error( 'pinecone_unavailable', esc_html__('Pinecone manager not available.', 'mxchat') );
1187		- }
1188		-
1189		- if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
1190		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
1191		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1192		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1193		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1194		- } else {
1195		- $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1196		- $api_key = $bot_config['api_key'] ?? '';
1197		- $host = $bot_config['host'] ?? '';
1198		- $namespace = $bot_config['namespace'] ?? '';
1199		- }
1200		-
1201		- if ( empty($host) || empty($api_key) ) {
1202		- return new WP_Error( 'pinecone_config', esc_html__('Pinecone not configured.', 'mxchat') );
1203		- }
1204		-
1205		- $base_id = md5( $source_url );
1206		- $vector_ids = array( $base_id );
1207		-
1208		- // NOTE: Pinecone's /vectors/list is a GET endpoint with query parameters.
1209		- // A POST is answered 200-with-an-empty-body, which reads as "no vectors".
1210		- $list_url = "https://{$host}/vectors/list";
1211		- $list_params = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
1212		- if ( ! empty($namespace) ) {
1213		- $list_params['namespace'] = $namespace;
1214		- }
1215		-
1216		- $list_resp = wp_remote_get( $list_url . '?' . http_build_query( $list_params ), array(
1217		- 'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ),
1218		- 'timeout' => 15,
1219		- ) );
1220		-
1221		- if ( ! is_wp_error($list_resp) ) {
1222		- $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
1223		- if ( ! empty($list_data['vectors']) ) {
1224		- foreach ( $list_data['vectors'] as $v ) {
1225		- $vector_ids[] = $v['id'];
1226		- }
	703	+
	704	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	705	+
	706	+ if (file_exists($pdf_path)) {
	707	+ wp_delete_file($pdf_path);
1227	708	}
1228	709	}
1229		-
1230		- // NOTE: /vectors/fetch is a GET endpoint too, and Pinecone expects the ids
1231		- // repeated (ids=a&ids=b) — http_build_query would emit ids[0]=a, so build
1232		- // the query string explicitly.
1233		- $fetch_query = array();
1234		- foreach ( $vector_ids as $fetch_vid ) {
1235		- $fetch_query[] = 'ids=' . rawurlencode( $fetch_vid );
1236		- }
1237		- if ( ! empty($namespace) ) {
1238		- $fetch_query[] = 'namespace=' . rawurlencode( $namespace );
1239		- }
1240		-
1241		- $fetch_resp = wp_remote_get( "https://{$host}/vectors/fetch?" . implode( '&', $fetch_query ), array(
1242		- 'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ),
1243		- 'timeout' => 15,
1244		- ) );
1245		-
1246		- if ( is_wp_error($fetch_resp) ) {
1247		- return new WP_Error( 'pinecone_fetch', esc_html__('Failed to fetch from Pinecone.', 'mxchat') );
1248		- }
1249		-
1250		- $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
1251		- $vectors = $fetch_data['vectors'] ?? array();
1252		-
1253		- if ( empty($vectors) ) {
1254		- return new WP_Error( 'not_found', esc_html__('Entry not found in Pinecone.', 'mxchat') );
1255		- }
1256		-
1257		- // Whitelisted metadata fields the spec calls out — shown so devs can confirm
1258		- // what is (and is NOT) stored per vector.
1259		- $meta_fields = array( 'text', 'source_url', 'type', 'last_updated', 'created_at', 'bot_id', 'chunk_index', 'total_chunks' );
1260		- $chunks = array();
1261		- $content_type = '';
1262		- foreach ( $vectors as $vid => $vector ) {
1263		- $meta = isset($vector['metadata']) && is_array($vector['metadata']) ? $vector['metadata'] : array();
1264		- $text = $meta['text'] ?? '';
1265		- $index = isset($meta['chunk_index']) ? intval($meta['chunk_index']) : count($chunks);
1266		- $content_type = $meta['type'] ?? $content_type;
1267		-
1268		- $clean_meta = array();
1269		- foreach ( $meta_fields as $field ) {
1270		- if ( array_key_exists( $field, $meta ) && $field !== 'text' ) {
1271		- $clean_meta[ $field ] = is_scalar( $meta[ $field ] ) ? (string) $meta[ $field ] : wp_json_encode( $meta[ $field ] );
1272		- }
1273		- }
1274		-
1275		- $chunks[] = array(
1276		- 'index' => $index,
1277		- 'text' => $text,
1278		- 'length' => function_exists('mb_strlen') ? mb_strlen( $text ) : strlen( $text ),
1279		- 'vector_id' => (string) $vid,
1280		- 'metadata' => $clean_meta,
1281		- );
1282		- }
1283		-
1284		- usort( $chunks, function( $a, $b ) { return $a['index'] - $b['index']; } );
1285		-
1286		- $assembled = implode( "\n\n", wp_list_pluck( $chunks, 'text' ) );
1287		-
1288		- return array(
1289		- 'store' => 'pinecone',
1290		- 'source_url' => $source_url,
1291		- 'content_type' => $content_type,
1292		- 'is_chunked' => count( $chunks ) > 1,
1293		- 'chunk_count' => count( $chunks ),
1294		- 'assembled' => $assembled,
1295		- 'assembled_length' => function_exists('mb_strlen') ? mb_strlen( $assembled ) : strlen( $assembled ),
1296		- 'chunks' => array_values( $chunks ),
1297		- 'metadata' => array(),
1298		- 'metadata_note' => esc_html__('Stored in Pinecone. Each chunk above lists the vector metadata fields actually present — if a field you expect (such as taxonomy terms) is missing here, it was not stored as metadata and is only searchable if it appears in the embedded text.', 'mxchat'),
1299		- );
1300	710	}
1301	711
1302		-/**
1303		- * AJAX: Save edited content — re-chunks and re-embeds as needed.
1304		- * Works for both WordPress DB and Pinecone entries.
1305		- */
1306		-public function ajax_mxchat_save_entry_content() {
1307		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
	712	+ public function mxchat_save_inline_prompt() {
	713	+ // Check for nonce security
	714	+ check_ajax_referer('mxchat_save_inline_nonce');
1308	715
1309		- if ( ! current_user_can('manage_options') ) {
1310		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
1311		- }
	716	+ // Verify permissions
	717	+ if (!current_user_can('manage_options')) {
	718	+ wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
	719	+ return;
	720	+ }
1312	721
1313		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
1314		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
1315		- $content = isset($_POST['content']) ? wp_kses_post( wp_unslash($_POST['content']) ) : '';
1316		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
1317		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1318		- $content_type = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : 'content';
	722	+ global $wpdb;
	723	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1319	724
1320		- if ( empty($content) ) {
1321		- wp_send_json_error( array( 'message' => 'Content cannot be empty.' ) );
1322		- }
	725	+ // Validate and sanitize input data
	726	+ $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
	727	+ $article_content = isset($_POST['article_content']) ? sanitize_textarea_field($_POST['article_content']) : '';
	728	+ $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
1323	729
1324		- // Get the embedding API key
1325		- $options = get_option('mxchat_options', array());
1326		- $api_key = '';
	730	+ if ($prompt_id > 0 && !empty($article_content)) {
	731	+ // Re-generate the embedding vector for the updated content
	732	+ $embedding_vector = $this->mxchat_generate_embedding($article_content);
1327	733
1328		- if ( $bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager') ) {
1329		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
1330		- $api_key = $bot_options['api_key'] ?? '';
1331		- }
1332		- if ( empty($api_key) ) {
1333		- $api_key = $options['api_key'] ?? '';
1334		- }
	734	+ if (is_array($embedding_vector)) {
	735	+ // Serialize the embedding vector before storing it
	736	+ $embedding_vector_serialized = serialize($embedding_vector);
1335	737
1336		- global $wpdb;
1337		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
	738	+ // Update the prompt in the database
	739	+ $updated = $wpdb->update(
	740	+ $table_name,
	741	+ array(
	742	+ 'article_content' => $article_content,
	743	+ 'embedding_vector' => $embedding_vector_serialized,
	744	+ 'source_url' => $article_url,
	745	+ ),
	746	+ array('id' => $prompt_id),
	747	+ array('%s', '%s', '%s'),
	748	+ array('%d')
	749	+ );
1338	750
1339		- // If source_url is empty but we have an entry_id, look it up
1340		- if ( empty($source_url) && $entry_id > 0 && $data_source === 'wordpress' ) {
1341		- $row = $wpdb->get_row( $wpdb->prepare( "SELECT source_url FROM {$table} WHERE id = %d", $entry_id ) );
1342		- if ( $row && ! empty($row->source_url) ) {
1343		- $source_url = $row->source_url;
1344		- }
1345		- }
	751	+ if ($updated !== false) {
	752	+ wp_send_json_success();
	753	+ } else {
	754	+ wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
	755	+ }
	756	+ } else {
	757	+ wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
	758	+ }
	759	+ } else {
	760	+ wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
	761	+ }
	762	+ }
1346	763
1347		- // For manual entries (no source_url or mxchat:// prefix), delete the old entry by ID first
1348		- // so submit_content_to_db creates a replacement instead of a duplicate
1349		- // Also treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
1350		- $is_legacy_manual = !empty($source_url) && strpos($source_url, 'mxchat.ai') !== false && strpos($source_url, 'mxchat://') !== 0;
1351		- if ( $entry_id > 0 && (empty($source_url) || strpos($source_url, 'mxchat://') === 0 || $is_legacy_manual) ) {
1352		- $wpdb->delete( $table, array( 'id' => $entry_id ), array( '%d' ) );
1353		- // Clear legacy URL so submit_content_to_db generates a unique mxchat:// identifier
1354		- // instead of reusing the shared URL (which would mass-delete other entries with the same URL)
1355		- if ( $is_legacy_manual ) {
1356		- $source_url = '';
1357		- }
1358		- }
1359	764
1360		- // Use the existing submit_content_to_db which handles chunking, Pinecone, and WP DB
1361		- $vector_id = ! empty($source_url) ? md5($source_url) : md5('mxchat_manual_' . $entry_id);
1362		-
1363		- // submit_content_to_db already handles: delete old chunks → re-chunk → re-embed → store
1364		- $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, $vector_id, $bot_id, $content_type );
1365		-
1366		- if ( is_wp_error($result) ) {
1367		- wp_send_json_error( array( 'message' => $result->get_error_message() ) );
1368		- }
1369		-
1370		- wp_send_json_success( array( 'message' => 'Content saved and re-embedded successfully.' ) );
1371		-}
1372		-
1373	765	public function mxchat_get_pdf_processing_status($pdf_url) {
1374	766	$pdf_url = esc_url_raw($pdf_url);
1375	767	$status = get_transient(sanitize_key('mxchat_pdf_status_' . md5($pdf_url)));
1376	768
		@@ -1413,18 +805,24 @@
1413	805	}
1414	806
1415	807
1416	808	public function mxchat_handle_sitemap_submission() {
	809	+ // Start logging the submission process
	810	+ //error_log('[MXCHAT-URL] ===== Starting URL submission process =====');
	811	+
1417	812	// Check if the form was submitted and verify permissions
1418	813	if (!isset($_POST['submit_sitemap']) || !current_user_can('manage_options')) {
	814	+ //error_log('[MXCHAT-URL] Error: Unauthorized access or form not submitted properly');
1419	815	wp_die(esc_html__('Unauthorized access', 'mxchat'));
1420	816	}
1421	817
1422	818	// Verify nonce
	819	+ //error_log('[MXCHAT-URL] Verifying nonce');
1423	820	check_admin_referer('mxchat_submit_sitemap_action', 'mxchat_submit_sitemap_nonce');
1424	821
1425	822	// Validate URL
1426	823	if (!isset($_POST['sitemap_url']) || empty($_POST['sitemap_url'])) {
	824	+ //error_log('[MXCHAT-URL] Error: Empty or missing URL');
1427	825	set_transient('mxchat_admin_notice_error',
1428	826	esc_html__('Please provide a valid URL.', 'mxchat'),
1429	827	30
1430	828	);
		@@ -1432,56 +830,43 @@
1432	830	exit;
1433	831	}
1434	832
1435	833	$submitted_url = esc_url_raw($_POST['sitemap_url']);
	834	+ //error_log('[MXCHAT-URL] Processing URL: ' . $submitted_url);
1436	835
1437		- // Convert Google Drive sharing URLs to direct download URLs
1438		- if ( strpos($submitted_url, 'drive.google.com') !== false ) {
1439		- $file_id = '';
1440		- if ( preg_match('/[?&]id=([a-zA-Z0-9_-]+)/', $submitted_url, $m) ) {
1441		- $file_id = $m[1];
1442		- } elseif ( preg_match('#/file/d/([a-zA-Z0-9_-]+)#', $submitted_url, $m) ) {
1443		- $file_id = $m[1];
1444		- }
1445		- if ( ! empty($file_id) ) {
1446		- $submitted_url = 'https://drive.google.com/uc?export=download&id=' . $file_id;
1447		- }
	836	+ // Validate API key first
	837	+ $options = get_option('mxchat_options');
	838	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	839	+
	840	+ if (strpos($selected_model, 'voyage') === 0) {
	841	+ $api_key = $options['voyage_api_key'] ?? '';
	842	+ $provider_name = 'Voyage AI';
	843	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	844	+ $api_key = $options['gemini_api_key'] ?? '';
	845	+ $provider_name = 'Google Gemini';
	846	+ } else {
	847	+ $api_key = $options['api_key'] ?? '';
	848	+ $provider_name = 'OpenAI';
1448	849	}
1449		-
1450		- // Get bot_id from form submission
1451		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1452		-
1453		- // Get bot-specific options and validate the embedding decision —
1454		- // custom-provider-aware (plan cbd5fd).
1455		- $bot_options = $this->get_bot_options($bot_id);
1456		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
1457		-
1458		- $preflight = MxChat_Utils::embedding_preflight($options);
1459		- if (!$preflight['ok']) {
1460		- set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
	850	+
	851	+ if (empty($api_key)) {
	852	+ $error_message = sprintf(
	853	+ esc_html__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
	854	+ $provider_name
	855	+ );
	856	+ //error_log('[MXCHAT-URL] Error: ' . $error_message);
	857	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	858	+ //error_log('[MXCHAT-URL] Set error transient: ' . $error_message);
1461	859	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1462	860	exit;
1463	861	}
1464		- $api_key = $preflight['api_key'];
1465	862
1466		- // Fetch URL — send an honest, versioned MXChat crawler UA (not a spoofed
1467		- // browser). Stale browser UAs are exactly what WAFs like SiteGround's
1468		- // ModSecurity flag as scrapers, 403-ing the fetch (including PDFs served
1469		- // from the site's own media library, which route through this same call).
1470		- // See mxchat_ingest_user_agent(). Accept is kept for content negotiation;
1471		- // the browser-only Accept-Language fingerprint is dropped so it stays
1472		- // coherent with a bot identity.
1473		- $response = wp_remote_get($submitted_url, array(
1474		- 'timeout' => 30,
1475		- 'sslverify' => false,
1476		- 'user-agent' => mxchat_ingest_user_agent(),
1477		- 'headers' => array(
1478		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1479		- ),
1480		- ));
	863	+ //error_log('[MXCHAT-URL] Fetching URL content');
	864	+ $response = wp_remote_get($submitted_url, array('timeout' => 30));
1481	865
1482	866	if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
1483	867	$error_message = is_wp_error($response) ? $response->get_error_message() : 'HTTP Status: ' . wp_remote_retrieve_response_code($response);
	868	+ //error_log('[MXCHAT-URL] Error fetching URL: ' . $error_message);
1484	869	set_transient('mxchat_admin_notice_error',
1485	870	sprintf(
1486	871	esc_html__('Failed to fetch the URL: %s', 'mxchat'),
1487	872	esc_html($error_message)
		@@ -1492,11 +877,13 @@
1492	877	exit;
1493	878	}
1494	879
1495	880	$content_type = wp_remote_retrieve_header($response, 'content-type');
	881	+ //error_log('[MXCHAT-URL] Content type: ' . $content_type);
1496	882	$body_content = wp_remote_retrieve_body($response);
1497	883
1498	884	if (empty($body_content)) {
	885	+ //error_log('[MXCHAT-URL] Error: Empty response body');
1499	886	set_transient('mxchat_admin_notice_error',
1500	887	esc_html__('Empty response received from URL.', 'mxchat'),
1501	888	30
1502	889	);
		@@ -1502,21 +889,29 @@
1502	889	);
1503	890	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1504	891	exit;
1505	892	}
	893	+ //error_log('[MXCHAT-URL] Retrieved body content length: ' . strlen($body_content) . ' bytes');
1506	894
1507	895	// Handle PDF URL
1508	896	if ($this->mxchat_is_pdf_url($submitted_url, $response)) {
1509		- $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response, $bot_id);
	897	+ //error_log('[MXCHAT-URL] Detected PDF URL, handling PDF for knowledge base');
	898	+ $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response);
	899	+ //error_log('[MXCHAT-URL] PDF handling result: ' . $result);
1510	900
1511		- if ($result === 'queued') {
1512		- set_transient('mxchat_admin_notice_success',
1513		- esc_html__('PDF queued for processing. Processing will start automatically.', 'mxchat'),
	901	+ if ($result === 'scheduled') {
	902	+ set_transient(
	903	+ 'mxchat_last_pdf_url',
	904	+ sanitize_text_field($submitted_url),
	905	+ DAY_IN_SECONDS
	906	+ );
	907	+ set_transient('mxchat_admin_notice_info',
	908	+ esc_html__('PDF processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1514	909	30
1515	910	);
1516	911	} else {
1517	912	set_transient('mxchat_admin_notice_error',
1518		- esc_html__('Failed to queue PDF processing: ', 'mxchat') . esc_html($result),
	913	+ esc_html__('Failed to start PDF processing: ', 'mxchat') . esc_html($result),
1519	914	30
1520	915	);
1521	916	}
1522	917
		@@ -1525,8 +920,9 @@
1525	920	}
1526	921
1527	922	// Handle Sitemap XML
1528	923	if (strpos($content_type, 'xml') !== false || strpos($body_content, '<urlset') !== false) {
	924	+ //error_log('[MXCHAT-URL] Detected XML content, processing as sitemap');
1529	925	libxml_use_internal_errors(true);
1530	926	$xml = simplexml_load_string($body_content);
1531	927	$xml_errors = libxml_get_errors();
1532	928	libxml_clear_errors();
		@@ -1531,8 +927,15 @@
1531	927	$xml_errors = libxml_get_errors();
1532	928	libxml_clear_errors();
1533	929
1534	930	if ($xml === false || !empty($xml_errors)) {
	931	+ //error_log('[MXCHAT-URL] Error: Invalid XML format');
	932	+ if (!empty($xml_errors)) {
	933	+ foreach ($xml_errors as $error) {
	934	+ //error_log('[MXCHAT-URL] XML Error: ' . $error->message);
	935	+ }
	936	+ }
	937	+
1535	938	set_transient('mxchat_admin_notice_error',
1536	939	esc_html__('Invalid sitemap XML. Please provide a valid sitemap.', 'mxchat'),
1537	940	30
1538	941	);
		@@ -1539,30 +942,29 @@
1539	942	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1540	943	exit;
1541	944	}
1542	945
1543		- $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url, $bot_id);
	946	+ //error_log('[MXCHAT-URL] Valid XML found, handling sitemap for knowledge base');
	947	+ $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url);
	948	+ //error_log('[MXCHAT-URL] Sitemap handling result: ' . $result);
1544	949
1545		- if ($result === 'queued') {
1546		- set_transient('mxchat_admin_notice_success',
1547		- esc_html__('Sitemap queued for processing. Processing will start automatically.', 'mxchat'),
	950	+ if ($result === 'scheduled') {
	951	+ set_transient(
	952	+ 'mxchat_last_sitemap_url',
	953	+ sanitize_text_field($submitted_url),
	954	+ DAY_IN_SECONDS
	955	+ );
	956	+ set_transient('mxchat_admin_notice_info',
	957	+ esc_html__('Sitemap processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1548	958	30
1549	959	);
1550	960	} else {
1551		- // Surface the reason the handler already computed (embedding pre-flight,
1552		- // empty sitemap, queue failure). The old message pointed at the status
1553		- // area, which is empty on this path — nothing was ever queued.
1554		- if (is_string($result) && $result !== '') {
1555		- set_transient('mxchat_admin_notice_error',
1556		- esc_html__('Failed to queue sitemap processing: ', 'mxchat') . esc_html($result),
1557		- 30
1558		- );
1559		- } else {
1560		- set_transient('mxchat_admin_notice_error',
1561		- esc_html__('Failed to queue sitemap processing.', 'mxchat'),
1562		- 30
1563		- );
1564		- }
	961	+ // Return to the admin page without a redirect for better error display
	962	+ // The error is already stored in the sitemap status transient
	963	+ set_transient('mxchat_admin_notice_error',
	964	+ esc_html__('Failed to start sitemap processing. Please check the status below for details.', 'mxchat'),
	965	+ 30
	966	+ );
1565	967	}
1566	968
1567	969	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1568	970	exit;
		@@ -1567,50 +969,127 @@
1567	969	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1568	970	exit;
1569	971	}
1570	972
1571		- // Handle Regular URL (single page)
	973	+ // Handle Regular URL
	974	+ //error_log('[MXCHAT-URL] Processing as regular webpage');
1572	975	$page_content = $this->mxchat_extract_main_content($body_content);
	976	+ //error_log('[MXCHAT-URL] Extracted content length: ' . strlen($page_content) . ' bytes');
	977	+
1573	978	$sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	979	+ //error_log('[MXCHAT-URL] Sanitized content length: ' . strlen($sanitized_content) . ' bytes');
1574	980
1575		- //error_log('[MXCHAT-URL-DEBUG] URL: ' . $submitted_url);
1576		- //error_log('[MXCHAT-URL-DEBUG] Extracted page_content length: ' . strlen($page_content));
1577		- //error_log('[MXCHAT-URL-DEBUG] Sanitized content length: ' . strlen($sanitized_content));
1578		- //error_log('[MXCHAT-URL-DEBUG] Content preview: ' . substr($sanitized_content, 0, 500));
1579		-
1580	981	if (empty($sanitized_content)) {
	982	+ //error_log('[MXCHAT-URL] Error: No valid content after sanitization');
	983	+
	984	+ // Set both transients - the error notice and the URL status
1581	985	set_transient('mxchat_admin_notice_error',
1582	986	esc_html__('No valid content found on the provided URL.', 'mxchat'),
1583	987	30
1584	988	);
	989	+
	990	+ // Set URL status transient
	991	+ set_transient('mxchat_single_url_status', [
	992	+ 'url' => $submitted_url,
	993	+ 'timestamp' => current_time('mysql'),
	994	+ 'status' => 'failed',
	995	+ 'error' => esc_html__('No valid content found on the provided URL.', 'mxchat')
	996	+ ], DAY_IN_SECONDS);
	997	+
1585	998	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1586	999	exit;
1587	1000	}
1588	1001
1589		- // For single URLs, process immediately using submit_content_to_db
1590		- // This handles chunking automatically for large content
1591		- $db_result = MxChat_Utils::submit_content_to_db(
1592		- $sanitized_content,
1593		- $submitted_url,
1594		- $api_key,
1595		- null,
1596		- $bot_id,
1597		- 'url' // content_type
1598		- );
	1002	+ //error_log('[MXCHAT-URL] Generating embedding for content');
	1003	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
1599	1004
1600		- if (is_wp_error($db_result)) {
1601		- $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1005	+ // Check if embedding_vector is a string (error message)
	1006	+ if (is_string($embedding_vector)) {
	1007	+ //error_log('[MXCHAT-URL] Error generating embedding: ' . $embedding_vector);
	1008	+ $error_message = esc_html__('Failed to generate embedding: ', 'mxchat') . esc_html($embedding_vector);
	1009	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1010	+
	1011	+ // Set both transients
1602	1012	set_transient('mxchat_admin_notice_error', $error_message, 30);
1603		- } else {
	1013	+
	1014	+ // Set URL status transient
	1015	+ set_transient('mxchat_single_url_status', [
	1016	+ 'url' => $submitted_url,
	1017	+ 'timestamp' => current_time('mysql'),
	1018	+ 'status' => 'failed',
	1019	+ 'error' => $error_message
	1020	+ ], DAY_IN_SECONDS);
	1021	+
	1022	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1023	+ exit;
	1024	+ }
	1025	+
	1026	+ if (is_array($embedding_vector)) {
	1027	+ //error_log('[MXCHAT-URL] Successfully generated embedding with ' . count($embedding_vector) . ' dimensions');
	1028	+
	1029	+ $db_result = MxChat_Utils::submit_content_to_db(
	1030	+ $sanitized_content,
	1031	+ $submitted_url,
	1032	+ $api_key
	1033	+ );
	1034	+
	1035	+ if (is_wp_error($db_result)) {
	1036	+ //error_log('[MXCHAT-URL] Error: Failed to store content in database: ' . $db_result->get_error_message());
	1037	+ $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1038	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1039	+
	1040	+ // Set both transients
	1041	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1042	+
	1043	+ // Set URL status transient
	1044	+ set_transient('mxchat_single_url_status', [
	1045	+ 'url' => $submitted_url,
	1046	+ 'timestamp' => current_time('mysql'),
	1047	+ 'status' => 'failed',
	1048	+ 'error' => $error_message
	1049	+ ], DAY_IN_SECONDS);
	1050	+
	1051	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1052	+ exit;
	1053	+ }
	1054	+
	1055	+ //error_log('[MXCHAT-URL] Successfully stored content in database');
1604	1056	$success_message = esc_html__('URL content successfully submitted!', 'mxchat');
	1057	+ //error_log('[MXCHAT-URL] Setting success transient: ' . $success_message);
	1058	+
	1059	+ // Set both transients
1605	1060	set_transient('mxchat_admin_notice_success', $success_message, 30);
	1061	+
	1062	+ // Set URL status transient with success
	1063	+ set_transient('mxchat_single_url_status', [
	1064	+ 'url' => $submitted_url,
	1065	+ 'timestamp' => current_time('mysql'),
	1066	+ 'status' => 'complete',
	1067	+ 'content_length' => strlen($sanitized_content),
	1068	+ 'embedding_dimensions' => count($embedding_vector)
	1069	+ ], DAY_IN_SECONDS);
	1070	+
	1071	+ } else {
	1072	+ //error_log('[MXCHAT-URL] Error: Failed to generate embedding. Unexpected result type: ' . gettype($embedding_vector));
	1073	+ $error_message = esc_html__('Failed to generate embedding: Unexpected result type. Please check your API key and try again.', 'mxchat');
	1074	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1075	+
	1076	+ // Set both transients
	1077	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1078	+
	1079	+ // Set URL status transient
	1080	+ set_transient('mxchat_single_url_status', [
	1081	+ 'url' => $submitted_url,
	1082	+ 'timestamp' => current_time('mysql'),
	1083	+ 'status' => 'failed',
	1084	+ 'error' => $error_message
	1085	+ ], DAY_IN_SECONDS);
1606	1086	}
1607	1087
	1088	+ //error_log('[MXCHAT-URL] ===== Completed URL submission process =====');
1608	1089	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1609	1090	exit;
1610	1091	}
1611		-
1612		-
1613	1092	public function mxchat_get_single_url_status() {
1614	1093	$status = get_transient('mxchat_single_url_status');
1615	1094	if (!$status) {
1616	1095	return null;
		@@ -1622,11 +1101,13 @@
1622	1101	}
1623	1102
1624	1103	return $status;
1625	1104	}
1626		-
1627		-public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url, $bot_id = 'default') {
	1105	+public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url) {
	1106	+ // Clear any single URL status when starting sitemap processing
	1107	+ delete_transient('mxchat_single_url_status');
1628	1108	if (!current_user_can('manage_options')) {
	1109	+ //error_log(esc_html__('Unauthorized sitemap processing attempt', 'mxchat'));
1629	1110	return false;
1630	1111	}
1631	1112
1632	1113	try {
		@@ -1635,228 +1116,331 @@
1635	1116	if (!$xml || !is_object($xml)) {
1636	1117	throw new Exception(__('Invalid XML object provided', 'mxchat'));
1637	1118	}
1638	1119
1639		- // Get bot-specific embedding API for validation
1640		- $bot_options = $this->get_bot_options($bot_id);
1641		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	1120	+ // Add embedding validation before processing
	1121	+ // Test embedding with a small sample text to verify API key is working
	1122	+ $test_result = $this->mxchat_generate_embedding("This is a test to verify the embedding API key is working.");
1642	1123
1643		- // Test the embedding API before processing
1644		- $test_phrase = "Test embedding generation for MxChat";
1645		- $test_result = $this->mxchat_generate_embedding($test_phrase, $bot_id);
1646		-
	1124	+ // Check if test_result is a string (error message) rather than an array (valid embedding)
1647	1125	if (is_string($test_result)) {
	1126	+ //error_log('[MXCHAT-URL] Embedding API validation failed: ' . $test_result);
	1127	+
	1128	+ // Store the error in the status transient so it can be displayed later
	1129	+ $status_data = array(
	1130	+ 'total_urls' => 0,
	1131	+ 'processed_urls' => 0,
	1132	+ 'status' => 'error',
	1133	+ 'error' => __('Embedding API validation failed: ', 'mxchat') . $test_result,
	1134	+ 'last_update' => time()
	1135	+ );
	1136	+
	1137	+ set_transient(
	1138	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1139	+ array_map('sanitize_text_field', $status_data),
	1140	+ DAY_IN_SECONDS
	1141	+ );
	1142	+
1648	1143	throw new Exception(__('Embedding API validation failed: ', 'mxchat') . $test_result);
1649	1144	}
1650	1145
	1146	+ // Make sure it's an array (valid embedding)
1651	1147	if (!is_array($test_result)) {
	1148	+ //error_log('[MXCHAT-URL] Embedding API returned unexpected result type: ' . gettype($test_result));
1652	1149	throw new Exception(__('Embedding API returned unexpected result type. Please check your configuration.', 'mxchat'));
1653	1150	}
1654	1151
1655		- // Extract URLs from sitemap
1656		- $urls = array();
	1152	+ $urls = [];
1657	1153	foreach ($xml->url as $url_element) {
1658	1154	$url = esc_url_raw((string)$url_element->loc);
1659	1155	if ($url) {
1660		- $urls[] = array('url' => $url);
	1156	+ $urls[] = $url;
1661	1157	}
1662	1158	}
1663	1159
1664		- $total_urls = count($urls);
	1160	+ $total_urls = absint(count($urls));
1665	1161
1666	1162	if ($total_urls < 1) {
1667	1163	throw new Exception(__('No valid URLs found in sitemap', 'mxchat'));
1668	1164	}
1669	1165
1670		- // Create unique queue ID
1671		- $queue_id = 'sitemap_' . md5($sitemap_url . time());
	1166	+ wp_schedule_single_event(time(), 'mxchat_process_sitemap_urls', array(
	1167	+ 'urls' => $urls,
	1168	+ 'sitemap_url' => $sitemap_url,
	1169	+ 'total_urls' => $total_urls,
	1170	+ 'batch_size' => absint(10),
	1171	+ 'batch_pause' => absint(5)
	1172	+ ));
1672	1173
1673		- // Add URLs to queue
1674		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'url', $urls, $bot_id);
	1174	+ $status_data = array(
	1175	+ 'total_urls' => $total_urls,
	1176	+ 'processed_urls' => 0,
	1177	+ 'status' => 'processing',
	1178	+ 'last_update' => time()
	1179	+ );
1675	1180
1676		- if ($queued_count === 0) {
1677		- throw new Exception(__('Failed to add URLs to processing queue', 'mxchat'));
1678		- }
	1181	+ set_transient(
	1182	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1183	+ array_map('sanitize_text_field', $status_data),
	1184	+ DAY_IN_SECONDS
	1185	+ );
1679	1186
1680		- // Store queue metadata
1681		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $sitemap_url);
1682		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'sitemap');
1683		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_urls);
1684		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
1685		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	1187	+ return __('scheduled', 'mxchat');
1686	1188
1687		- // Store queue ID in transient for status tracking
1688		- set_transient('mxchat_active_queue_sitemap', $queue_id, DAY_IN_SECONDS);
1689		- set_transient('mxchat_last_sitemap_url', $sitemap_url, DAY_IN_SECONDS);
1690		-
1691		- return 'queued';
1692		-
1693		- } catch (Exception $e) {
	1189	+ } catch (\Exception $e) {
1694	1190	$error_message = $e->getMessage();
1695	1191	//error_log(sprintf(esc_html__('Error preparing sitemap for processing: %s', 'mxchat'), esc_html($error_message)));
1696	1192
	1193	+ // Store the sitemap URL and error in transients so they can be displayed
	1194	+ set_transient(
	1195	+ 'mxchat_last_sitemap_url',
	1196	+ sanitize_text_field($sitemap_url),
	1197	+ DAY_IN_SECONDS
	1198	+ );
	1199	+
	1200	+ $status_data = array(
	1201	+ 'total_urls' => 0,
	1202	+ 'processed_urls' => 0,
	1203	+ 'status' => 'error',
	1204	+ 'error' => $error_message,
	1205	+ 'last_update' => time()
	1206	+ );
	1207	+
	1208	+ set_transient(
	1209	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1210	+ array_map('sanitize_text_field', $status_data),
	1211	+ DAY_IN_SECONDS
	1212	+ );
	1213	+
1697	1214	return $error_message;
1698	1215	}
1699		-
1700	1216	}
1701	1217
1702		-/**
1703		- * Remove shortcode tags but preserve the content inside them
1704		- * Example: [vc_column]Hello World[/vc_column] becomes "Hello World"
1705		- *
1706		- * @param string $content The content containing shortcodes
1707		- * @return string Content with shortcode tags removed but inner content preserved
1708		- */
1709		-/**
1710		- * Single-pass HTML entity decode for text entering the knowledge base.
1711		- * The corpus should hold what a human reads: a stored `&amp;` consumes
1712		- * extra tokens, distorts the vector away from the form a visitor's
1713		- * question uses, and can be quoted back verbatim in an answer.
1714		- * Deliberately NOT looped to a fixed point — a stored `&amp;amp;` is a
1715		- * legitimate literal `&amp;` and must not collapse further (data loss).
1716		- * UTF-8 charset keeps multibyte (CJK/RTL) text untouched. Both assembly
1717		- * paths call this at their output points so the treatment cannot drift.
1718		- * (Plan d2c92e.)
1719		- */
1720		-private function mxchat_decode_entities_for_indexing($text) {
1721		- return html_entity_decode((string) $text, ENT_QUOTES | ENT_HTML5, 'UTF-8');
1722		-}
	1218	+public function mxchat_process_sitemap_urls_cron($urls, $sitemap_url, $total_urls, $batch_size, $batch_pause) {
	1219	+ // Validate inputs
	1220	+ $sitemap_url = esc_url_raw($sitemap_url);
	1221	+ $total_urls = absint($total_urls);
	1222	+ $batch_size = absint($batch_size);
	1223	+ $batch_pause = absint($batch_pause);
1723	1224
1724		-/**
1725		- * Price lines for a product's indexed text, pinned to the store's BASE currency.
1726		- *
1727		- * The four product assembly paths each used to call get_woocommerce_currency_symbol()
1728		- * with no argument, which resolves the currency active on the CURRENT request.
1729		- * Multi-currency plugins (CURCY, WOOCS, Aelia, WPML Multicurrency) filter that per
1730		- * request, so whichever currency the store happened to be serving when an import ran
1731		- * was frozen into every product it indexed. The amounts have the mirror problem: the
1732		- * woocommerce_product_get_* filters convert prices in the 'view' context but not in
1733		- * 'edit', so a converted amount could be paired with an unconverted symbol and produce
1734		- * a price that is not merely wrong but incoherent.
1735		- *
1736		- * Base currency option + 'edit' context makes both halves agree and makes the output
1737		- * independent of when the import ran. The currency CODE is emitted alongside the symbol
1738		- * so the model cannot read a bare "$" as USD on a store whose plugin swapped symbols.
1739		- * (Plan 7403ec.)
1740		- */
1741		-private function mxchat_product_price_lines($product) {
1742		- if (!is_object($product) || !method_exists($product, 'get_regular_price')) {
1743		- return '';
	1225	+ if (!is_array($urls) || empty($urls)) {
	1226	+ return;
1744	1227	}
1745	1228
1746		- $currency = get_option('woocommerce_currency');
1747		- $currency = is_string($currency) ? trim($currency) : '';
1748		- $symbol = ($currency !== '')
1749		- ? get_woocommerce_currency_symbol($currency)
1750		- : get_woocommerce_currency_symbol();
1751		- $symbol = $this->mxchat_decode_entities_for_indexing($symbol);
	1229	+ try {
	1230	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	1231	+ $status = get_transient($status_key);
1752	1232
1753		- $regular_price = $product->get_regular_price('edit');
1754		- $sale_price = $product->get_sale_price('edit');
1755		- $price = $product->get_price('edit');
	1233	+ if (!$status || !is_array($status)) {
	1234	+ throw new Exception('Invalid status data retrieved from transient');
	1235	+ }
1756	1236
1757		- $lines = '';
	1237	+ // Initialize failed_urls array if it doesn't exist
	1238	+ if (!isset($status['failed_urls_list']) || !is_array($status['failed_urls_list'])) {
	1239	+ $status['failed_urls_list'] = [];
	1240	+ }
1758	1241
1759		- if (!empty($regular_price)) {
1760		- $lines .= "Price: " . $this->mxchat_format_indexed_price($regular_price, $currency, $symbol) . "\n";
1761		- } elseif (!empty($price)) {
1762		- $lines .= "Price: " . $this->mxchat_format_indexed_price($price, $currency, $symbol) . "\n";
1763		- }
	1242	+ $start_url = absint($status['processed_urls']);
	1243	+ $end_url = min($start_url + $batch_size, $total_urls);
1764	1244
1765		- if (!empty($sale_price) && $sale_price !== $regular_price) {
1766		- $lines .= "Sale Price: " . $this->mxchat_format_indexed_price($sale_price, $currency, $symbol) . "\n";
1767		- }
	1245	+ // Track batch statistics
	1246	+ $batch_stats = [
	1247	+ 'processed' => 0,
	1248	+ 'failed' => 0,
	1249	+ 'last_error' => '',
	1250	+ 'embedding_errors' => 0
	1251	+ ];
1768	1252
1769		- if ($product->is_type('variable')) {
1770		- list($min_price, $max_price) = $this->mxchat_variation_price_range($product);
1771		- if ($min_price !== null && $max_price !== null && (float) $min_price !== (float) $max_price) {
1772		- $lines .= "Price Range: " . $this->mxchat_format_indexed_price($min_price, $currency, $symbol)
1773		- . " - " . $this->mxchat_format_indexed_price($max_price, $currency, $symbol) . "\n";
	1253	+ // Check embedding configuration with first URL (only on first batch)
	1254	+ if ($start_url === 0) {
	1255	+ $test_url = esc_url_raw($urls[0]);
	1256	+ $test_response = wp_remote_get($test_url);
	1257	+
	1258	+ if (!is_wp_error($test_response) && wp_remote_retrieve_response_code($test_response) === 200) {
	1259	+ $test_html = wp_remote_retrieve_body($test_response);
	1260	+ $test_content = $this->mxchat_extract_main_content($test_html);
	1261	+ $test_sanitized = $this->mxchat_sanitize_content_for_api($test_content);
	1262	+
	1263	+ if (!empty($test_sanitized)) {
	1264	+ $test_embedding = $this->mxchat_generate_embedding($test_sanitized);
	1265	+
	1266	+ if (is_string($test_embedding)) {
	1267	+ throw new Exception('Embedding generation failed: ' . $test_embedding);
	1268	+ }
	1269	+
	1270	+ if (!is_array($test_embedding)) {
	1271	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($test_embedding));
	1272	+ }
	1273	+ }
	1274	+ }
1774	1275	}
1775		- }
1776	1276
1777		- return $lines;
1778		-}
	1277	+ for ($i = $start_url; $i < $end_url; $i++) {
	1278	+ $page_url = esc_url_raw($urls[$i]);
	1279	+ $max_retries = 3;
	1280	+ $retry_count = 0;
	1281	+ $url_processed = false;
	1282	+ $last_error = '';
1779	1283
1780		-/**
1781		- * One indexed price amount, labelled with its currency code.
1782		- *
1783		- * "INR 1299.00 (Rs.1299.00)" — the code is what the model should reason from; the symbol
1784		- * is kept so a quoted price still reads naturally. Falls back to the old symbol-only
1785		- * shape when WooCommerce has no base currency configured, and drops the parenthetical
1786		- * when the symbol is absent or IS the code (several currencies have no distinct glyph).
1787		- */
1788		-private function mxchat_format_indexed_price($amount, $currency, $symbol) {
1789		- $amount = (string) $amount;
	1284	+ while (!$url_processed && $retry_count < $max_retries) {
	1285	+ try {
	1286	+ // Attempt to fetch the URL
	1287	+ $page_response = wp_remote_get($page_url, array('timeout' => 30));
1790	1288
1791		- if ($currency === '') {
1792		- return $symbol . $amount;
1793		- }
	1289	+ if (is_wp_error($page_response)) {
	1290	+ throw new Exception('HTTP request failed: ' . $page_response->get_error_message());
	1291	+ }
1794	1292
1795		- if ($symbol === '' || $symbol === $currency) {
1796		- return $currency . ' ' . $amount;
1797		- }
	1293	+ $response_code = wp_remote_retrieve_response_code($page_response);
	1294	+ if ($response_code !== 200) {
	1295	+ throw new Exception('HTTP Status: ' . $response_code);
	1296	+ }
1798	1297
1799		- return $currency . ' ' . $amount . ' (' . $symbol . $amount . ')';
1800		-}
	1298	+ $page_html = wp_remote_retrieve_body($page_response);
	1299	+
	1300	+ if (empty($page_html)) {
	1301	+ throw new Exception('Empty response body');
	1302	+ }
1801	1303
1802		-/**
1803		- * Min/max variation price read from the variations themselves in 'edit' context.
1804		- *
1805		- * get_variation_price() reads WooCommerce's display price cache, which multi-currency
1806		- * plugins populate with converted values — the same defect the rest of this helper
1807		- * exists to remove. Returns raw stored strings (not floats) so the indexed text keeps
1808		- * the store's own price formatting, and (null, null) when no variation carries a price.
1809		- */
1810		-private function mxchat_variation_price_range($product) {
1811		- $min_raw = null;
1812		- $max_raw = null;
1813		- $min_val = null;
1814		- $max_val = null;
	1304	+ $page_content = $this->mxchat_extract_main_content($page_html);
	1305	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
1815	1306
1816		- $children = method_exists($product, 'get_children') ? $product->get_children() : array();
	1307	+ if (empty($sanitized_content)) {
	1308	+ throw new Exception('No valid content found after processing');
	1309	+ }
1817	1310
1818		- foreach ($children as $child_id) {
1819		- $variation = wc_get_product($child_id);
1820		- if (!$variation) {
1821		- continue;
	1311	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	1312	+
	1313	+ if (is_string($embedding_vector)) {
	1314	+ throw new Exception('Embedding generation failed: ' . $embedding_vector);
	1315	+ }
	1316	+
	1317	+ if (!is_array($embedding_vector)) {
	1318	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($embedding_vector));
	1319	+ }
	1320	+
	1321	+ // Submit to database
	1322	+ $options = get_option('mxchat_options');
	1323	+ $submission_result = MxChat_Utils::submit_content_to_db($sanitized_content, $page_url, $options['api_key']);
	1324	+
	1325	+ if (is_wp_error($submission_result)) {
	1326	+ throw new Exception('Database submission failed: ' . $submission_result->get_error_message());
	1327	+ }
	1328	+
	1329	+ // Success!
	1330	+ $url_processed = true;
	1331	+ $batch_stats['processed']++;
	1332	+
	1333	+ } catch (Exception $e) {
	1334	+ $retry_count++;
	1335	+ $last_error = $e->getMessage();
	1336	+
	1337	+ //error_log("URL {$page_url} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	1338	+
	1339	+ // Track embedding errors specifically
	1340	+ if (strpos($last_error, 'Embedding') !== false) {
	1341	+ $batch_stats['embedding_errors']++;
	1342	+ }
	1343	+
	1344	+ if ($retry_count < $max_retries) {
	1345	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	1346	+ sleep(pow(2, $retry_count - 1));
	1347	+ }
	1348	+ }
	1349	+ }
	1350	+
	1351	+ // If URL still not processed after all retries, mark as failed
	1352	+ if (!$url_processed) {
	1353	+ $batch_stats['failed']++;
	1354	+ $batch_stats['last_error'] = $last_error;
	1355	+
	1356	+ // Add to failed URLs list
	1357	+ $status['failed_urls_list'][] = [
	1358	+ 'url' => $page_url,
	1359	+ 'error' => $last_error,
	1360	+ 'time' => time(),
	1361	+ 'retries' => $max_retries
	1362	+ ];
	1363	+
	1364	+ // Limit the number of failed URLs we store to prevent transient size issues
	1365	+ if (count($status['failed_urls_list']) > 100) {
	1366	+ $status['failed_urls_list'] = array_slice($status['failed_urls_list'], -100);
	1367	+ }
	1368	+ }
	1369	+
	1370	+ // Update progress
	1371	+ $status['processed_urls'] = absint($i + 1);
	1372	+ $status['last_update'] = time();
	1373	+ $status['failed_urls'] = absint($status['failed_urls'] ?? 0) + ($url_processed ? 0 : 1);
	1374	+ $status['last_error'] = $batch_stats['last_error'];
	1375	+
	1376	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1377	+
	1378	+ // If we have too many consecutive embedding errors, stop processing
	1379	+ if ($batch_stats['embedding_errors'] >= 10) {
	1380	+ throw new Exception('Too many consecutive embedding failures detected. Please check your API configuration.');
	1381	+ }
1822	1382	}
1823		- $raw = $variation->get_price('edit');
1824		- if ($raw === '' || $raw === null) {
1825		- continue;
	1383	+
	1384	+ // If all URLs in this batch failed, stop processing
	1385	+ if ($batch_stats['processed'] === 0 && $batch_stats['failed'] > 0) {
	1386	+ $status['status'] = 'error';
	1387	+ $status['error'] = sprintf(
	1388	+ 'Processing stopped: %d consecutive failures in batch. Last error: %s',
	1389	+ $batch_stats['failed'],
	1390	+ $batch_stats['last_error']
	1391	+ );
	1392	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1393	+ return;
1826	1394	}
1827		- $val = (float) $raw;
1828		- if ($min_val === null || $val < $min_val) {
1829		- $min_val = $val;
1830		- $min_raw = $raw;
	1395	+
	1396	+ // Update final progress
	1397	+ $status['processed_urls'] = min($end_url, $total_urls);
	1398	+ $status['last_update'] = time();
	1399	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1400	+
	1401	+ // Check if we've processed all URLs
	1402	+ if ($end_url >= $total_urls) {
	1403	+ // All URLs have been processed - mark as complete
	1404	+ $status['status'] = 'complete';
	1405	+ $status['processed_urls'] = $total_urls;
	1406	+
	1407	+ // Add completion summary
	1408	+ $status['completion_summary'] = [
	1409	+ 'total_urls' => $total_urls,
	1410	+ 'successful_urls' => $total_urls - absint($status['failed_urls'] ?? 0),
	1411	+ 'failed_urls' => absint($status['failed_urls'] ?? 0),
	1412	+ 'completion_time' => current_time('mysql')
	1413	+ ];
	1414	+
	1415	+ // Save the completed status (don't delete it - let user dismiss manually)
	1416	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1417	+
	1418	+ // DON'T delete the status transients here - let user dismiss manually
	1419	+ } else {
	1420	+ // Schedule next batch
	1421	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_sitemap_urls', array(
	1422	+ 'urls' => $urls,
	1423	+ 'sitemap_url' => $sitemap_url,
	1424	+ 'total_urls' => $total_urls,
	1425	+ 'batch_size' => $batch_size,
	1426	+ 'batch_pause' => $batch_pause,
	1427	+ ));
1831	1428	}
1832		- if ($max_val === null || $val > $max_val) {
1833		- $max_val = $val;
1834		- $max_raw = $raw;
1835		- }
	1429	+ } catch (\Exception $e) {
	1430	+ $status['status'] = 'error';
	1431	+ $status['error'] = $e->getMessage();
	1432	+ set_transient($status_key, $status, DAY_IN_SECONDS);
1836	1433	}
1837		-
1838		- return array($min_raw, $max_raw);
1839	1434	}
1840		-
1841		-private function strip_shortcode_tags_preserve_content($content) {
1842		- // Single-pass regex removes all shortcode brackets: [tag], [tag attr="val"], [/tag], [tag /]
1843		- // Content between tags is inherently preserved since only brackets are targeted
1844		- $result = preg_replace('/\[\/?\w[\w-]*[^\]]*\]/', '', $content);
1845		- return ($result !== null) ? $result : $content;
1846		-}
1847		-
1848	1435	public function mxchat_sanitize_content_for_api($content) {
1849	1436	//error_log('[MXCHAT-SANITIZE] Original content preview: ' . substr($content, 0, 500) . '...');
1850		-
1851		- // Remove shortcode tags but PRESERVE content inside them
1852		- $content = $this->strip_shortcode_tags_preserve_content($content);
1853		-
	1437	+
1854	1438	// Remove script, style tags, and HTML comments
1855	1439	$content = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $content);
1856	1440	$content = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $content);
1857	1441	$content = preg_replace('/<!--(.|\s)*?-->/', '', $content);
1858		-
	1442	+
1859	1443	// Remove all HTML tags and decode HTML entities
1860	1444	$content = wp_strip_all_tags($content);
1861	1445	$content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5);
1862	1446
		@@ -1882,50 +1466,24 @@
1882	1466
1883	1467	// Ensure valid UTF-8 encoding
1884	1468	$content = wp_check_invalid_utf8($content);
1885	1469
1886		- // Remove extremely long runs with no whitespace (base64 blobs, minified JS).
1887		- // Counts CHARACTERS (/u), and never strips a run containing characters from a
1888		- // script written without spaces — Japanese, Chinese, Thai, Khmer, Lao, Myanmar —
1889		- // where a normal paragraph is legitimately one unbroken run.
1890		- $content = preg_replace_callback('/\S{300,}/u', function ($m) {
1891		- return preg_match('/[\p{Han}\p{Hiragana}\p{Katakana}\p{Thai}\p{Khmer}\p{Lao}\p{Myanmar}]/u', $m[0]) ? $m[0] : ' ';
1892		- }, $content);
1893		-
1894		- // Remove emoji/symbol blocks only — not the whole supplementary plane, which
1895		- // also holds CJK Extension B ideographs used in real Chinese/Japanese names
1896		- $content = preg_replace('/[\x{1F000}-\x{1F0FF}\x{1F300}-\x{1FAFF}]/u', '', $content);
	1470	+ // Remove any extremely long strings without spaces (often garbage)
	1471	+ $content = preg_replace('/\S{300,}/', ' ', $content);
1897	1472
	1473	+ // Replace problematic characters that often cause database issues
	1474	+ $content = preg_replace('/[\x{10000}-\x{10FFFF}]/u', '', $content); // Remove emoji and other high Unicode characters
	1475	+
1898	1476	// Replace any remaining potentially problematic characters with spaces
1899	1477	// BUT preserve newlines by temporarily replacing them
1900		- //
1901		- // \p{Sc} (Symbol, currency) is in the allowlist because every currency sign —
1902		- // $ € £ ¥ ₹ — is Sc, not Sm, and without it this pass silently replaced every
1903		- // one of them with a space. That hit far more than product prices: any indexed
1904		- // page quoting "$4.99" was embedded as " 4.99", leaving the model no way to know
1905		- // which currency (or that it was money at all). Found while verifying plan 7403ec.
1906		- //
1907		- // \p{M} (Mark) is in the allowlist because combining marks are not decoration —
1908		- // they are letters' other half. Arabic harakat, Hebrew niqqud, and above all the
1909		- // Devanagari vowel signs and virama (Mc/Mn) are mandatory in their scripts. Each
1910		- // one used to be replaced by a SPACE, which split one word into several fragments
1911		- // and turned Indic text into gibberish. Decomposed (NFD) Latin lost every accent
1912		- // the same way. Invisible in English, which is why it went unreported for so long.
1913		- //
1914		- // \p{So} (Symbol, other) covers ™ © ® ° ✓ — meaning-bearing marks that were also
1915		- // becoming spaces ("Brand® name" indexed as "Brand name", "200°C" as "200 C").
1916		- // Emoji are ALSO So: they stay stripped by the emoji-block pass immediately above,
1917		- // which runs BEFORE this line. That ordering is load-bearing now — moving this
1918		- // line above the emoji strip would let emoji back into the index. Plan a19914.
1919	1478	$content = str_replace("\n", "NEWLINE_PLACEHOLDER", $content);
1920		- $content = preg_replace('/[^\p{L}\p{N}\p{P}\p{Z}\p{Sm}\p{Sc}\p{M}\p{So}]/u', ' ', $content);
	1479	+ $content = preg_replace('/[^\p{L}\p{N}\p{P}\p{Z}\p{Sm}]/u', ' ', $content);
1921	1480	$content = str_replace("NEWLINE_PLACEHOLDER", "\n", $content);
1922	1481
1923		- // Limit to reasonable length if needed (byte limit — MySQL TEXT is byte-sized,
1924		- // but cut on a character boundary so a multibyte char is never split mid-sequence)
	1482	+ // Limit to reasonable length if needed
1925	1483	$max_length = 65000; // Just under MySQL TEXT field limit
1926	1484	if (strlen($content) > $max_length) {
1927		- $content = mb_strcut($content, 0, $max_length, 'UTF-8');
	1485	+ $content = substr($content, 0, $max_length);
1928	1486	}
1929	1487
1930	1488	//error_log('[MXCHAT-SANITIZE] Sanitized content preview: ' . substr($content, 0, 500) . '...');
1931	1489	return $content;
		@@ -1940,12 +1498,12 @@
1940	1498	@$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'), LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
1941	1499	$xpath = new DOMXPath($dom);
1942	1500
1943	1501	// For debugging purposes
1944		- $debugEnabled = true; // Set to true to enable debugging output
	1502	+ $debugEnabled = false; // Set to true to enable debugging output
1945	1503	$debug = function($message) use ($debugEnabled) {
1946	1504	if ($debugEnabled) {
1947		- //error_log('[MXCHAT-EXTRACT-DEBUG] ' . $message);
	1505	+ //error_log('[MXCHAT-DEBUG] ' . $message);
1948	1506	}
1949	1507	};
1950	1508
1951	1509	// Direct targeting for Gerow theme posts
		@@ -2044,25 +1602,18 @@
2044	1602	'//main',
2045	1603	'//div[contains(@class, "content")]'
2046	1604	];
2047	1605
2048		- // First handle Elementor content - get only leaf widget containers to avoid duplicates
	1606	+ // First handle Elementor content
2049	1607	$debug("Checking for Elementor content");
2050		- // Get widget containers that are direct children of widgets (not nested inside other widget containers)
2051		- $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-widget")]//div[contains(@class, "elementor-widget-container")]');
	1608	+ $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-element")]//div[contains(@class, "elementor-widget-container")]');
2052	1609	if ($elementor_widgets && $elementor_widgets->length > 0) {
2053	1610	$debug("Found Elementor widgets");
2054		- $seen_content = array(); // Track seen content to avoid duplicates
2055	1611	$combined_content = '';
2056	1612	foreach ($elementor_widgets as $widget) {
2057	1613	$widget_content = $dom->saveHTML($widget);
2058	1614	if (!empty($widget_content)) {
2059		- // Create a hash of the content to detect duplicates
2060		- $content_hash = md5($widget_content);
2061		- if (!isset($seen_content[$content_hash])) {
2062		- $seen_content[$content_hash] = true;
2063		- $combined_content .= $widget_content;
2064		- }
	1615	+ $combined_content .= $widget_content;
2065	1616	}
2066	1617	}
2067	1618	if (!empty($combined_content)) {
2068	1619	$debug("Returning Elementor content");
		@@ -2074,14 +1625,15 @@
2074	1625	foreach ($selectors as $selector) {
2075	1626	$debug("Trying selector: " . $selector);
2076	1627	$nodes = $xpath->query($selector);
2077	1628	if ($nodes && $nodes->length > 0) {
2078		- $debug("Found " . $nodes->length . " matches for selector: " . $selector);
2079		- // Only take the FIRST matching node to avoid duplicate content
2080		- // (pages often have nested or multiple containers with same class)
2081		- $content = $dom->saveHTML($nodes->item(0));
	1629	+ $debug("Found matches for selector: " . $selector);
	1630	+ $content = '';
	1631	+ foreach ($nodes as $node) {
	1632	+ $content .= $dom->saveHTML($node);
	1633	+ }
2082	1634	if (!empty($content)) {
2083		- $debug("Returning content from selector: " . $selector . " (first match only)");
	1635	+ $debug("Returning content from selector: " . $selector);
2084	1636	return $content;
2085	1637	}
2086	1638	}
2087	1639	}
		@@ -2105,106 +1657,16 @@
2105	1657	$debug("Returning blog-area section content");
2106	1658	return $content;
2107	1659	}
2108	1660	}
2109		-
2110		- // Generic container selectors for non-CMS sites (like .asp pages)
2111		- $debug("Trying generic container selectors");
2112		- $generic_selectors = [
2113		- '//div[@id="main"]',
2114		- '//div[@id="wrapper"]',
2115		- '//div[@id="page"]',
2116		- '//div[@id="site-content"]',
2117		- '//div[contains(@class, "main-content")]',
2118		- '//div[contains(@class, "page-content")]',
2119		- '//div[contains(@class, "site-content")]',
2120		- ];
2121		-
2122		- foreach ($generic_selectors as $selector) {
2123		- $debug("Trying generic selector: " . $selector);
2124		- $nodes = $xpath->query($selector);
2125		- if ($nodes && $nodes->length > 0) {
2126		- $content = $dom->saveHTML($nodes->item(0));
2127		- if (!empty($content)) {
2128		- $debug("Returning content from generic selector: " . $selector);
2129		- return $content;
2130		- }
2131		- }
2132		- }
2133		-
2134		- // Paragraph-based content detection - find regions with substantial text
2135		- $debug("Trying paragraph-based content detection");
2136		- $paragraphs = $xpath->query('//p[string-length(normalize-space()) > 50]');
2137		- if ($paragraphs && $paragraphs->length >= 3) {
2138		- $debug("Found " . $paragraphs->length . " substantial paragraphs");
2139		- // Collect all substantial paragraphs and their content
2140		- $paragraph_content = '';
2141		- foreach ($paragraphs as $p) {
2142		- $paragraph_content .= $dom->saveHTML($p) . "\n";
2143		- }
2144		- if (!empty($paragraph_content)) {
2145		- $debug("Returning paragraph-based content");
2146		- return $paragraph_content;
2147		- }
2148		- }
2149		-
2150		- // Improved body fallback - strip nav/header/footer elements first
2151		- $debug("Using improved body fallback");
	1661	+
	1662	+ // Fallback: Return the body content if no specific selector matches
	1663	+ $debug("Using body fallback");
2152	1664	$body = $dom->getElementsByTagName('body');
2153	1665	if ($body->length > 0) {
2154		- // Clone the body to avoid modifying the original DOM
2155		- $body_clone = $body->item(0)->cloneNode(true);
2156		-
2157		- // Remove common non-content elements by tag name
2158		- $remove_tags = ['nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript'];
2159		- foreach ($remove_tags as $tag) {
2160		- $elements = $body_clone->getElementsByTagName($tag);
2161		- // Iterate backwards to safely remove elements
2162		- for ($i = $elements->length - 1; $i >= 0; $i--) {
2163		- $el = $elements->item($i);
2164		- if ($el && $el->parentNode) {
2165		- $el->parentNode->removeChild($el);
2166		- }
2167		- }
2168		- }
2169		-
2170		- // Remove elements with common non-content class names using XPath on the cloned body
2171		- $temp_dom = new DOMDocument();
2172		- @$temp_dom->appendChild($temp_dom->importNode($body_clone, true));
2173		- $temp_xpath = new DOMXPath($temp_dom);
2174		-
2175		- $remove_class_patterns = [
2176		- '//*[contains(@class, "nav")]',
2177		- '//*[contains(@class, "menu")]',
2178		- '//*[contains(@class, "sidebar")]',
2179		- '//*[contains(@class, "footer")]',
2180		- '//*[contains(@class, "header")]',
2181		- '//*[contains(@id, "nav")]',
2182		- '//*[contains(@id, "menu")]',
2183		- '//*[contains(@id, "sidebar")]',
2184		- '//*[contains(@id, "footer")]',
2185		- '//*[contains(@id, "header")]',
2186		- ];
2187		-
2188		- foreach ($remove_class_patterns as $pattern) {
2189		- $elements = $temp_xpath->query($pattern);
2190		- if ($elements) {
2191		- for ($i = $elements->length - 1; $i >= 0; $i--) {
2192		- $el = $elements->item($i);
2193		- if ($el && $el->parentNode) {
2194		- $el->parentNode->removeChild($el);
2195		- }
2196		- }
2197		- }
2198		- }
2199		-
2200		- $cleaned_content = $temp_dom->saveHTML();
2201		- if (!empty($cleaned_content)) {
2202		- $debug("Returning cleaned body content");
2203		- return $cleaned_content;
2204		- }
	1666	+ return $dom->saveHTML($body->item(0));
2205	1667	}
2206		-
	1668	+
2207	1669	// Last resort: return the original HTML
2208	1670	$debug("Returning original HTML");
2209	1671	return $html;
2210	1672	} catch (Exception $e) {
		@@ -2256,41 +1718,47 @@
2256	1718	try {
2257	1719	// Verify the request
2258	1720	check_ajax_referer('mxchat_status_nonce', 'nonce');
2259	1721
2260		- // Get active queue IDs
2261		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
2262		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
	1722	+ // Get the status just like in your admin page
	1723	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1724	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
2263	1725
2264		- $sitemap_status = false;
2265		- $pdf_status = false;
	1726	+ $pdf_status = $pdf_url ? $this->mxchat_get_pdf_processing_status($pdf_url) : false;
	1727	+ $sitemap_status = $sitemap_url ? $this->mxchat_get_sitemap_processing_status($sitemap_url) : false;
2266	1728
2267		- // Get sitemap queue status
2268		- if ($sitemap_queue_id) {
2269		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
	1729	+ // Add the PDF URL to the status object
	1730	+ if ($pdf_status && $pdf_url) {
	1731	+ $pdf_status['pdf_url'] = $pdf_url;
2270	1732	}
2271	1733
2272		- // Get PDF queue status
2273		- if ($pdf_queue_id) {
2274		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
2275		- }
	1734	+ // Set the current PDF URL for the manual batch processing button
	1735	+ $current_pdf_url = $pdf_url;
2276	1736
	1737	+ // Check for true processing status, not just presence of status
2277	1738	$is_active_processing =
2278		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
2279		- ($pdf_status && $pdf_status['status'] === 'processing');
	1739	+ ($sitemap_status && isset($sitemap_status['status']) && $sitemap_status['status'] === 'processing') ||
	1740	+ ($pdf_status && isset($pdf_status['status']) && $pdf_status['status'] === 'processing');
2280	1741
	1742	+ // Get single URL status, but only if no processing is active
	1743	+ $single_url_status = !$is_active_processing ? $this->mxchat_get_single_url_status() : false;
	1744	+
	1745	+ // REMOVED: Auto-clearing of completed status - now only done via dismiss button
	1746	+
2281	1747	// Return JSON response with the status data
2282	1748	wp_send_json(array(
2283	1749	'pdf_status' => $pdf_status,
2284	1750	'sitemap_status' => $sitemap_status,
	1751	+ 'single_url_status' => $single_url_status,
2285	1752	'is_processing' => $is_active_processing,
2286		- 'sitemap_queue_id' => $sitemap_queue_id,
2287		- 'pdf_queue_id' => $pdf_queue_id
	1753	+ 'current_pdf_url' => $current_pdf_url
2288	1754	));
2289	1755
2290	1756	} catch (Exception $e) {
	1757	+ // Log the error
2291	1758	//error_log('MxChat Status Update Error: ' . $e->getMessage());
2292	1759
	1760	+ // Return a friendly error response
2293	1761	wp_send_json_error(array(
2294	1762	'message' => 'Error getting status updates: ' . $e->getMessage(),
2295	1763	'status' => 'error'
2296	1764	));
		@@ -2295,1371 +1763,8 @@
2295	1763	'status' => 'error'
2296	1764	));
2297	1765	}
2298	1766	}
2299		-
2300		-/**
2301		- * Helper function to get queue status data
2302		- */
2303		-private function mxchat_get_queue_status_data($queue_id, $type = 'sitemap') {
2304		- global $wpdb;
2305		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
2306		-
2307		- // Get counts by status
2308		- $counts = $wpdb->get_results($wpdb->prepare(
2309		- "SELECT status, COUNT(*) as count
2310		- FROM $table_name
2311		- WHERE queue_id = %s
2312		- GROUP BY status",
2313		- $queue_id
2314		- ), OBJECT_K);
2315		-
2316		- $total = 0;
2317		- $completed = 0;
2318		- $failed = 0;
2319		- $processing = 0;
2320		- $pending = 0;
2321		-
2322		- foreach ($counts as $status => $data) {
2323		- $count = absint($data->count);
2324		- $total += $count;
2325		-
2326		- switch ($status) {
2327		- case 'completed':
2328		- $completed = $count;
2329		- break;
2330		- case 'failed':
2331		- $failed = $count;
2332		- break;
2333		- case 'processing':
2334		- $processing = $count;
2335		- break;
2336		- case 'pending':
2337		- $pending = $count;
2338		- break;
2339		- }
2340		- }
2341		-
2342		- if ($total === 0) {
2343		- return false;
2344		- }
2345		-
2346		- // Calculate percentage
2347		- $percentage = round((($completed + $failed) / $total) * 100);
2348		-
2349		- // Get failed items details (limit to 50)
2350		- $failed_items = array();
2351		- if ($failed > 0) {
2352		- $failed_results = $wpdb->get_results($wpdb->prepare(
2353		- "SELECT item_type, item_data, error_message, attempts, completed_at
2354		- FROM $table_name
2355		- WHERE queue_id = %s
2356		- AND status = 'failed'
2357		- AND attempts >= max_attempts
2358		- ORDER BY id DESC
2359		- LIMIT 50",
2360		- $queue_id
2361		- ));
2362		-
2363		- foreach ($failed_results as $item) {
2364		- $data = json_decode($item->item_data, true);
2365		- $url = $type === 'pdf' ? ($data['pdf_url'] ?? '') : ($data['url'] ?? '');
2366		-
2367		- $failed_items[] = array(
2368		- 'url' => $url,
2369		- 'page' => $type === 'pdf' ? ($data['page_number'] ?? 0) : 0,
2370		- 'error' => $item->error_message,
2371		- 'retries' => $item->attempts,
2372		- 'time' => strtotime($item->completed_at)
2373		- );
2374		- }
2375		- }
2376		-
2377		- // Get queue metadata
2378		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
2379		-
2380		- // Determine if queue is complete
2381		- $is_complete = ($pending === 0 && $processing === 0);
2382		-
2383		- // Get last update time
2384		- $last_update = $wpdb->get_var($wpdb->prepare(
2385		- "SELECT MAX(UNIX_TIMESTAMP(COALESCE(completed_at, started_at, created_at)))
2386		- FROM $table_name
2387		- WHERE queue_id = %s",
2388		- $queue_id
2389		- ));
2390		-
2391		- $last_update_text = $last_update ? human_time_diff($last_update, time()) . ' ' . __('ago', 'mxchat') : __('Just now', 'mxchat');
2392		-
2393		- // Format based on type
2394		- if ($type === 'pdf') {
2395		- return array(
2396		- 'total_pages' => $total,
2397		- 'processed_pages' => $completed + $failed,
2398		- 'failed_pages' => $failed,
2399		- 'percentage' => $percentage,
2400		- 'status' => $is_complete ? 'complete' : 'processing',
2401		- 'last_update' => $last_update_text,
2402		- 'failed_pages_list' => $failed_items,
2403		- 'pdf_url' => $source_url,
2404		- 'queue_id' => $queue_id
2405		- );
2406		- } else {
2407		- return array(
2408		- 'total_urls' => $total,
2409		- 'processed_urls' => $completed + $failed,
2410		- 'failed_urls' => $failed,
2411		- 'percentage' => $percentage,
2412		- 'status' => $is_complete ? 'complete' : 'processing',
2413		- 'last_update' => $last_update_text,
2414		- 'failed_urls_list' => $failed_items,
2415		- 'sitemap_url' => $source_url,
2416		- 'queue_id' => $queue_id
2417		- );
2418		- }
2419		-}
2420		-
2421		-/**
2422		- * Public method to get processing status for both sitemap and PDF queues
2423		- * Used by admin pages to display processing status
2424		- *
2425		- * @return array Array with 'sitemap_status', 'pdf_status', and 'is_processing' keys
2426		- */
2427		-public function mxchat_get_processing_statuses() {
2428		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
2429		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
2430		-
2431		- $sitemap_status = false;
2432		- $pdf_status = false;
2433		-
2434		- if ($sitemap_queue_id) {
2435		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
2436		- }
2437		-
2438		- if ($pdf_queue_id) {
2439		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
2440		- }
2441		-
2442		- $is_processing =
2443		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
2444		- ($pdf_status && $pdf_status['status'] === 'processing');
2445		-
2446		- return array(
2447		- 'sitemap_status' => $sitemap_status,
2448		- 'pdf_status' => $pdf_status,
2449		- 'is_processing' => $is_processing
2450		- );
2451		-}
2452		-
2453		-/**
2454		- * AJAX handler to get recent knowledge entries for real-time table updates
2455		- * UPDATED: Now supports both WordPress DB and Pinecone data sources
2456		- */
2457		-public function ajax_mxchat_get_recent_entries() {
2458		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2459		-
2460		- if (!current_user_can('manage_options')) {
2461		- wp_send_json_error(array('message' => 'Unauthorized'));
2462		- return;
2463		- }
2464		-
2465		- global $wpdb;
2466		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2467		-
2468		- // Get parameters
2469		- $last_id = isset($_POST['last_id']) ? absint($_POST['last_id']) : 0;
2470		- $limit = isset($_POST['limit']) ? min(absint($_POST['limit']), 50) : 10;
2471		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2472		-
2473		- // Check if Pinecone is enabled for this bot
2474		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
2475		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
2476		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2477		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
2478		-
2479		- if ($use_pinecone && $has_pinecone_api) {
2480		- // PINECONE DATA SOURCE - Get grouped entry count (not raw vector count)
2481		- // Use mxchat_fetch_pinecone_records which returns total_unique_entries
2482		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, '', 1, 10, $bot_id, '');
2483		- $total_count = $records['total'] ?? 0;
2484		-
2485		- // For Pinecone, we don't return individual entries during polling
2486		- // (entries are already displayed on page load via mxchat_fetch_pinecone_records)
2487		- // We just return the updated count
2488		- wp_send_json_success(array(
2489		- 'entries' => array(),
2490		- 'total_count' => absint($total_count),
2491		- 'max_id' => $last_id,
2492		- 'data_source' => 'pinecone'
2493		- ));
2494		- return;
2495		- }
2496		-
2497		- // WORDPRESS DB DATA SOURCE
2498		- // Build query to get entries newer than last_id
2499		- $where_clauses = array('1=1');
2500		- $where_values = array();
2501		-
2502		- if ($last_id > 0) {
2503		- $where_clauses[] = 'id > %d';
2504		- $where_values[] = $last_id;
2505		- }
2506		-
2507		- // Note: WordPress DB table doesn't have bot_id column
2508		- // Multi-bot filtering is handled via Pinecone namespaces
2509		-
2510		- $where_sql = implode(' AND ', $where_clauses);
2511		-
2512		- // Get recent entries
2513		- $query = "SELECT id, article_content, source_url, timestamp
2514		- FROM $table_name
2515		- WHERE $where_sql
2516		- ORDER BY id DESC
2517		- LIMIT %d";
2518		-
2519		- $where_values[] = $limit;
2520		-
2521		- $entries = $wpdb->get_results($wpdb->prepare($query, $where_values));
2522		-
2523		- // Get total count of GROUPED entries (by source_url) - matches pagination display
2524		- // Count unique source_urls (excluding mxchat:// internal refs) + count of ungrouped rows
2525		- $total_count = $wpdb->get_var(
2526		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2527		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
2528		- );
2529		-
2530		- // Format entries for response
2531		- $formatted_entries = array();
2532		- $preview_length = 150;
2533		- foreach ($entries as $entry) {
2534		- // Parse chunk metadata using the proper chunker method (same as initial page load)
2535		- if (class_exists('MxChat_Chunker')) {
2536		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($entry->article_content);
2537		- $display_content = $chunk_meta['text'];
2538		- $chunk_metadata = $chunk_meta['metadata'];
2539		- } else {
2540		- $display_content = $entry->article_content;
2541		- $chunk_metadata = array();
2542		- }
2543		-
2544		- $content_preview = mb_strlen($display_content) > $preview_length
2545		- ? mb_substr($display_content, 0, $preview_length) . '...'
2546		- : $display_content;
2547		-
2548		- $formatted_entries[] = array(
2549		- 'id' => $entry->id,
2550		- 'preview' => esc_html($content_preview),
2551		- 'full_content' => wp_kses_post(wpautop($display_content)),
2552		- 'content_length' => mb_strlen($display_content),
2553		- 'preview_length' => $preview_length,
2554		- 'source_url' => $entry->source_url,
2555		- 'has_link' => !empty($entry->source_url) && strpos($entry->source_url, 'mxchat://') !== 0,
2556		- 'chunk_metadata' => $chunk_metadata,
2557		- 'bot_id' => $entry->bot_id ?? 'default',
2558		- 'edit_nonce' => wp_create_nonce('mxchat_edit_entry_nonce'),
2559		- 'delete_nonce' => wp_create_nonce('mxchat_delete_prompt_nonce')
2560		- );
2561		- }
2562		-
2563		- wp_send_json_success(array(
2564		- 'entries' => $formatted_entries,
2565		- 'total_count' => absint($total_count),
2566		- 'max_id' => !empty($entries) ? $entries[0]->id : $last_id,
2567		- 'data_source' => 'wordpress'
2568		- ));
2569		-}
2570		-
2571		-/**
2572		- * Get Pinecone total count from stats API
2573		- * Helper function for ajax_mxchat_get_recent_entries
2574		- */
2575		-private function mxchat_get_pinecone_count_from_stats($pinecone_options) {
2576		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2577		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
2578		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
2579		-
2580		- if (empty($api_key) || empty($host)) {
2581		- return 0;
2582		- }
2583		-
2584		- try {
2585		- $stats_url = "https://{$host}/describe_index_stats";
2586		-
2587		- $response = wp_remote_post($stats_url, array(
2588		- 'headers' => array(
2589		- 'Api-Key' => $api_key,
2590		- 'Content-Type' => 'application/json'
2591		- ),
2592		- 'body' => '{}',
2593		- 'timeout' => 10
2594		- ));
2595		-
2596		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2597		- $body = wp_remote_retrieve_body($response);
2598		- $stats_data = json_decode($body, true);
2599		-
2600		- // If namespace is specified, get count from that specific namespace
2601		- if (!empty($namespace) && isset($stats_data['namespaces'][$namespace]['vectorCount'])) {
2602		- return intval($stats_data['namespaces'][$namespace]['vectorCount']);
2603		- }
2604		-
2605		- // If no namespace specified or namespace not found in response, use total
2606		- return intval($stats_data['totalVectorCount'] ?? 0);
2607		- }
2608		-
2609		- return 0;
2610		-
2611		- } catch (Exception $e) {
2612		- return 0;
2613		- }
2614		-}
2615		-
2616		-/**
2617		- * AJAX handler to refresh Pinecone entries table via AJAX
2618		- * Returns the table HTML for updating the UI without a full page reload
2619		- */
2620		-public function ajax_mxchat_refresh_pinecone_entries() {
2621		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2622		-
2623		- if (!current_user_can('manage_options')) {
2624		- wp_send_json_error(array('message' => 'Unauthorized'));
2625		- return;
2626		- }
2627		-
2628		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2629		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2630		- $per_page = 25;
2631		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2632		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2633		-
2634		- // Get Pinecone manager and options
2635		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
2636		- if (!$pinecone_manager) {
2637		- wp_send_json_error(array('message' => 'Pinecone manager not available'));
2638		- return;
2639		- }
2640		-
2641		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
2642		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2643		- $pinecone_api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2644		-
2645		- if (!$use_pinecone || empty($pinecone_api_key)) {
2646		- wp_send_json_error(array('message' => 'Pinecone not configured'));
2647		- return;
2648		- }
2649		-
2650		- // Fetch records from Pinecone
2651		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, $search_query, $page, $per_page, $bot_id, $content_type_filter);
2652		- $prompts = $records['data'] ?? array();
2653		- $total_records = $records['total'] ?? 0;
2654		-
2655		- // Preprocess Pinecone records — set chunk_metadata and display_content
2656		- // (matches admin-knowledge-page.php preprocessing)
2657		- foreach ($prompts as $prompt) {
2658		- if (isset($prompt->chunk_index) && $prompt->chunk_index !== null) {
2659		- $prompt->chunk_metadata = array(
2660		- 'chunk_index' => intval($prompt->chunk_index),
2661		- 'total_chunks' => isset($prompt->total_chunks) ? intval($prompt->total_chunks) : null,
2662		- 'is_chunked' => isset($prompt->is_chunked) ? (bool) $prompt->is_chunked : true,
2663		- 'source_url' => $prompt->source_url ?? ''
2664		- );
2665		- $prompt->display_content = $prompt->article_content;
2666		- } else {
2667		- $prompt->chunk_metadata = array();
2668		- $prompt->display_content = $prompt->article_content ?? '';
2669		- }
2670		- }
2671		-
2672		- // Group prompts by source_url
2673		- $grouped_prompts = array();
2674		- foreach ($prompts as $prompt) {
2675		- $source_url = '';
2676		- if (!empty($prompt->chunk_metadata['source_url'])) {
2677		- $source_url = $prompt->chunk_metadata['source_url'];
2678		- } elseif (!empty($prompt->source_url)) {
2679		- $source_url = $prompt->source_url;
2680		- }
2681		-
2682		- if (!empty($source_url)) {
2683		- if (!isset($grouped_prompts[$source_url])) {
2684		- $grouped_prompts[$source_url] = array();
2685		- }
2686		- $grouped_prompts[$source_url][] = $prompt;
2687		- } else {
2688		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2689		- }
2690		- }
2691		-
2692		- // Sort each group by chunk_index
2693		- foreach ($grouped_prompts as $source_url => &$group) {
2694		- usort($group, function($a, $b) {
2695		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2696		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2697		- return $index_a - $index_b;
2698		- });
2699		- }
2700		- unset($group);
2701		-
2702		- // Build HTML for the table rows - must match admin-knowledge-page.php structure exactly
2703		- ob_start();
2704		- $display_index = 0;
2705		- $current_page = $page;
2706		- $data_source = 'pinecone';
2707		- $current_bot_id = $bot_id;
2708		- $preview_length = 150;
2709		-
2710		- if (empty($grouped_prompts)) {
2711		- echo '<tr><td colspan="4" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2712		- esc_html_e('No knowledge entries found in Pinecone.', 'mxchat');
2713		- echo '</td></tr>';
2714		- } else {
2715		- foreach ($grouped_prompts as $source_url => $group) {
2716		- $chunk_count = count($group);
2717		- $first_prompt = $group[0];
2718		- $display_index++;
2719		-
2720		- if ($chunk_count > 1) {
2721		- // Multiple chunks - show grouped row with expand button
2722		- $group_id = 'group-' . md5($source_url);
2723		- ?>
2724		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2725		- class="mxchat-chunk-group-header"
2726		- data-source="<?php echo esc_attr($data_source); ?>"
2727		- data-group-id="<?php echo esc_attr($group_id); ?>"
2728		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2729		- <td style="padding: 12px 16px; text-align: center;">
2730		- <input type="checkbox"
2731		- class="mxchat-entry-checkbox"
2732		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2733		- data-source="<?php echo esc_attr($data_source); ?>"
2734		- data-source-url="<?php echo esc_attr($source_url); ?>"
2735		- data-is-group="true"
2736		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2737		- </td>
2738		- <td style="padding: 12px 16px; font-size: 13px;">
2739		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2740		- </td>
2741		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2742		- <div class="mxchat-chunk-group-info">
2743		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2744		- <span class="dashicons dashicons-arrow-right-alt2"></span>
2745		- </button>
2746		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2747		- <span class="mxchat-chunk-preview">
2748		- <?php
2749		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : (isset($first_prompt->article_content) ? $first_prompt->article_content : '');
2750		- $content_preview = mb_substr($parent_content, 0, 100);
2751		- echo esc_html($content_preview . '...');
2752		- ?>
2753		- </span>
2754		- </div>
2755		- </td>
2756		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2757		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2758		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2759		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2760		- <?php esc_html_e('View Source', 'mxchat'); ?>
2761		- </a>
2762		- <?php else : ?>
2763		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2764		- <?php endif; ?>
2765		- </td>
2766		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2767		- <?php if ($data_source !== 'pinecone') : ?>
2768		- <button type="button"
2769		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2770		- data-source-url="<?php echo esc_attr($source_url); ?>"
2771		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2772		- data-data-source="<?php echo esc_attr($data_source); ?>"
2773		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2774		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2775		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2776		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2777		- </button>
2778		- <?php endif; ?>
2779		- <button type="button"
2780		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2781		- data-source-url="<?php echo esc_attr($source_url); ?>"
2782		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2783		- data-data-source="<?php echo esc_attr($data_source); ?>"
2784		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2785		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2786		- style="color: var(--mxch-error);"
2787		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2788		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2789		- </button>
2790		- </td>
2791		- </tr>
2792		- <?php
2793		- // Render hidden chunk rows
2794		- foreach ($group as $chunk_index => $chunk) {
2795		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
2796		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
2797		- $content = isset($chunk->display_content) ? $chunk->display_content : (isset($chunk->article_content) ? $chunk->article_content : '');
2798		- $content_preview = mb_strlen($content) > $preview_length
2799		- ? mb_substr($content, 0, $preview_length) . '...'
2800		- : $content;
2801		- ?>
2802		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
2803		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
2804		- data-source="<?php echo esc_attr($data_source); ?>"
2805		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
2806		- <td style="padding: 12px 16px; text-align: center;">
2807		- <!-- Checkbox column placeholder for chunks (managed by group) -->
2808		- </td>
2809		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
2810		- <!-- Hidden ID column for chunks -->
2811		- </td>
2812		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2813		- <div class="mxchat-accordion-wrapper">
2814		- <div class="mxchat-content-preview">
2815		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
2816		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
2817		- </span>
2818		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2819		- <?php if (mb_strlen($content) > $preview_length) : ?>
2820		- <button class="mxchat-expand-toggle" type="button">
2821		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2822		- </button>
2823		- <?php endif; ?>
2824		- </div>
2825		- <div class="mxchat-content-full" style="display: none;">
2826		- <div class="content-view">
2827		- <?php
2828		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2829		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2830		- echo wp_kses_post(wpautop($content));
2831		- echo '</div>';
2832		- } else {
2833		- echo wp_kses_post(wpautop($content));
2834		- }
2835		- ?>
2836		- </div>
2837		- </div>
2838		- </div>
2839		- </td>
2840		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2841		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
2842		- </td>
2843		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
2844		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
2845		- </td>
2846		- </tr>
2847		- <?php
2848		- }
2849		- } else {
2850		- // Single entry - display normally with accordion
2851		- $prompt = $first_prompt;
2852		- $content = isset($prompt->display_content) ? $prompt->display_content : (isset($prompt->article_content) ? $prompt->article_content : '');
2853		- $content_preview = mb_strlen($content) > $preview_length
2854		- ? mb_substr($content, 0, $preview_length) . '...'
2855		- : $content;
2856		- ?>
2857		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
2858		- data-source="<?php echo esc_attr($data_source); ?>"
2859		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2860		- <td style="padding: 12px 16px; text-align: center;">
2861		- <input type="checkbox"
2862		- class="mxchat-entry-checkbox"
2863		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2864		- data-source="<?php echo esc_attr($data_source); ?>"
2865		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
2866		- data-is-group="false"
2867		- data-chunk-count="1">
2868		- </td>
2869		- <td style="padding: 12px 16px; font-size: 13px;">
2870		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2871		- </td>
2872		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2873		- <div class="mxchat-accordion-wrapper">
2874		- <div class="mxchat-content-preview">
2875		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2876		- <?php if (mb_strlen($content) > $preview_length) : ?>
2877		- <button class="mxchat-expand-toggle" type="button">
2878		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2879		- </button>
2880		- <?php endif; ?>
2881		- </div>
2882		- <div class="mxchat-content-full" style="display: none;">
2883		- <div class="content-view">
2884		- <?php
2885		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2886		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2887		- echo wp_kses_post(wpautop($content));
2888		- echo '</div>';
2889		- } else {
2890		- echo wp_kses_post(wpautop($content));
2891		- }
2892		- ?>
2893		- </div>
2894		- </div>
2895		- </div>
2896		- </td>
2897		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2898		- <?php
2899		- $actual_source = $source_url;
2900		- if (strpos($source_url, '_ungrouped_') === 0) {
2901		- $actual_source = $prompt->source_url ?? '';
2902		- }
2903		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
2904		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2905		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2906		- <?php esc_html_e('View', 'mxchat'); ?>
2907		- </a>
2908		- <?php else : ?>
2909		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
2910		- <?php endif; ?>
2911		- </td>
2912		- <td style="padding: 12px 16px;">
2913		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-ajax" data-vector-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_pinecone_prompt_nonce'); ?>" style="color: var(--mxch-error);">
2914		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2915		- </button>
2916		- </td>
2917		- </tr>
2918		- <?php
2919		- }
2920		- }
2921		- }
2922		- $html = ob_get_clean();
2923		-
2924		- // Generate pagination HTML for Pinecone
2925		- $total_pages = ceil($total_records / $per_page);
2926		- $pagination_html = '';
2927		- if ($total_pages > 1) {
2928		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
2929		-
2930		- // Previous button
2931		- if ($page > 1) {
2932		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
2933		- }
2934		-
2935		- // Page numbers
2936		- $start_page = max(1, $page - 2);
2937		- $end_page = min($total_pages, $page + 2);
2938		-
2939		- if ($start_page > 1) {
2940		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
2941		- if ($start_page > 2) {
2942		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2943		- }
2944		- }
2945		-
2946		- for ($i = $start_page; $i <= $end_page; $i++) {
2947		- if ($i == $page) {
2948		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
2949		- } else {
2950		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
2951		- }
2952		- }
2953		-
2954		- if ($end_page < $total_pages) {
2955		- if ($end_page < $total_pages - 1) {
2956		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2957		- }
2958		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
2959		- }
2960		-
2961		- // Next button
2962		- if ($page < $total_pages) {
2963		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
2964		- }
2965		-
2966		- $pagination_html .= '</div>';
2967		- }
2968		-
2969		- wp_send_json_success(array(
2970		- 'html' => $html,
2971		- 'pagination_html' => $pagination_html,
2972		- 'total_count' => $total_records,
2973		- 'total_pages' => $total_pages,
2974		- 'page' => $page,
2975		- 'per_page' => $per_page,
2976		- 'data_source' => 'pinecone'
2977		- ));
2978		-}
2979		-
2980		-/**
2981		- * AJAX handler for pagination - handles both WordPress DB and Pinecone data sources
2982		- * Returns paginated entries without requiring a full page reload
2983		- */
2984		-public function ajax_mxchat_paginate_entries() {
2985		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2986		-
2987		- if (!current_user_can('manage_options')) {
2988		- wp_send_json_error(array('message' => 'Unauthorized'));
2989		- return;
2990		- }
2991		-
2992		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2993		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2994		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2995		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2996		- $per_page = 25;
2997		-
2998		- // Check if Pinecone is enabled for this bot
2999		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
3000		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
3001		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3002		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
3003		-
3004		- if ($use_pinecone && $has_pinecone_api) {
3005		- // Delegate to Pinecone pagination handler (pass search params)
3006		- $_POST['page'] = $page;
3007		- $_POST['search'] = $search_query;
3008		- $_POST['content_type'] = $content_type_filter;
3009		- $this->ajax_mxchat_refresh_pinecone_entries();
3010		- return;
3011		- }
3012		-
3013		- // WordPress DB pagination - MUST match initial page load logic exactly
3014		- global $wpdb;
3015		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3016		- $offset = ($page - 1) * $per_page;
3017		-
3018		- // Build WHERE clause for search and content type filtering
3019		- $where_clauses = array();
3020		- $where_values = array();
3021		-
3022		- if ($search_query) {
3023		- $where_clauses[] = "article_content LIKE %s";
3024		- $where_values[] = '%' . $wpdb->esc_like($search_query) . '%';
3025		- }
3026		-
3027		- if ($content_type_filter) {
3028		- switch ($content_type_filter) {
3029		- case 'manual':
3030		- $where_clauses[] = "(source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')";
3031		- break;
3032		- case 'pdf':
3033		- $where_clauses[] = "source_url LIKE '%.pdf'";
3034		- break;
3035		- case 'url':
3036		- $where_clauses[] = "source_url != '' AND source_url IS NOT NULL AND source_url NOT LIKE 'mxchat://%' AND source_url NOT LIKE '%.pdf'";
3037		- break;
3038		- }
3039		- }
3040		-
3041		- $where_sql = !empty($where_clauses) ? 'WHERE ' . implode(' AND ', $where_clauses) : '';
3042		-
3043		- // Count grouped entries with filters applied
3044		- if (!empty($where_values)) {
3045		- $count_args = array_merge($where_values, $where_values);
3046		- $count_query = $wpdb->prepare(
3047		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
3048		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))",
3049		- ...$count_args
3050		- );
3051		- $total_records = $wpdb->get_var($count_query);
3052		- } else if (!empty($where_sql)) {
3053		- // Content type filter only (no search), no prepared values needed
3054		- $total_records = $wpdb->get_var(
3055		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
3056		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))"
3057		- );
3058		- } else {
3059		- // No filters
3060		- $total_records = $wpdb->get_var(
3061		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
3062		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
3063		- );
3064		- }
3065		- $total_pages = ceil($total_records / $per_page);
3066		-
3067		- // Step 1: Get unique source_urls for this page (ordered by latest timestamp) with filters
3068		- if (!empty($where_values)) {
3069		- $query_args = array_merge($where_values, array($per_page, $offset));
3070		- $urls_query = $wpdb->prepare(
3071		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
3072		- {$where_sql}
3073		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
3074		- ...$query_args
3075		- );
3076		- } else if (!empty($where_sql)) {
3077		- $urls_query = $wpdb->prepare(
3078		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
3079		- {$where_sql}
3080		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
3081		- $per_page, $offset
3082		- );
3083		- } else {
3084		- $urls_query = $wpdb->prepare(
3085		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
3086		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
3087		- $per_page, $offset
3088		- );
3089		- }
3090		- $page_urls = $wpdb->get_results($urls_query);
3091		-
3092		- // Step 2: Build list of source_urls to fetch
3093		- $url_list = array();
3094		- $url_order_map = array();
3095		- $order_index = 0;
3096		- foreach ($page_urls as $url_row) {
3097		- $url = $url_row->source_url;
3098		- $url_list[] = $url;
3099		- $url_order_map[$url] = $order_index++;
3100		- }
3101		-
3102		- // Step 3: Fetch all rows for these source_urls (with search filter if applicable)
3103		- $prompts = array();
3104		- if (!empty($url_list)) {
3105		- $placeholders = implode(',', array_fill(0, count($url_list), '%s'));
3106		- if ($search_query) {
3107		- // Include search filter in the final fetch
3108		- $prompts_query = $wpdb->prepare(
3109		- "SELECT id, article_content, source_url, timestamp, role_restriction
3110		- FROM {$table_name}
3111		- WHERE source_url IN ($placeholders) AND article_content LIKE %s
3112		- ORDER BY timestamp DESC",
3113		- ...array_merge($url_list, ['%' . $wpdb->esc_like($search_query) . '%'])
3114		- );
3115		- } else {
3116		- $prompts_query = $wpdb->prepare(
3117		- "SELECT id, article_content, source_url, timestamp, role_restriction
3118		- FROM {$table_name}
3119		- WHERE source_url IN ($placeholders)
3120		- ORDER BY timestamp DESC",
3121		- $url_list
3122		- );
3123		- }
3124		- $prompts = $wpdb->get_results($prompts_query);
3125		- }
3126		-
3127		- // Group prompts by source_url for chunk display
3128		- $grouped_prompts = array();
3129		- foreach ($prompts as $prompt) {
3130		- $source_url = $prompt->source_url ?? '';
3131		-
3132		- // Parse chunk metadata using the proper chunker method (same as initial page load)
3133		- if (class_exists('MxChat_Chunker')) {
3134		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($prompt->article_content);
3135		- $prompt->chunk_metadata = $chunk_meta['metadata'];
3136		- $prompt->display_content = $chunk_meta['text'];
3137		- } else {
3138		- $prompt->chunk_metadata = array();
3139		- $prompt->display_content = $prompt->article_content;
3140		- }
3141		-
3142		- if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0) {
3143		- if (!isset($grouped_prompts[$source_url])) {
3144		- $grouped_prompts[$source_url] = array();
3145		- }
3146		- $grouped_prompts[$source_url][] = $prompt;
3147		- } else {
3148		- // Ungrouped entries
3149		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
3150		- }
3151		- }
3152		-
3153		- // Sort groups by the original URL order (newest first)
3154		- uksort($grouped_prompts, function($a, $b) use ($url_order_map) {
3155		- $order_a = isset($url_order_map[$a]) ? $url_order_map[$a] : PHP_INT_MAX;
3156		- $order_b = isset($url_order_map[$b]) ? $url_order_map[$b] : PHP_INT_MAX;
3157		- return $order_a - $order_b;
3158		- });
3159		-
3160		- // Sort each group internally by chunk_index
3161		- foreach ($grouped_prompts as $source_url => &$group) {
3162		- usort($group, function($a, $b) {
3163		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
3164		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
3165		- return $index_a - $index_b;
3166		- });
3167		- }
3168		- unset($group);
3169		-
3170		- // Build HTML for the table rows
3171		- ob_start();
3172		- $display_index = 0;
3173		- $current_page = $page;
3174		- $data_source = 'wordpress';
3175		- $current_bot_id = $bot_id;
3176		- $preview_length = 150;
3177		-
3178		- if (empty($grouped_prompts)) {
3179		- echo '<tr><td colspan="5" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
3180		- esc_html_e('No knowledge entries found. Use the Import Options to add content.', 'mxchat');
3181		- echo '</td></tr>';
3182		- } else {
3183		- foreach ($grouped_prompts as $source_url => $group) {
3184		- $chunk_count = count($group);
3185		- $first_prompt = $group[0];
3186		- $display_index++;
3187		-
3188		- if ($chunk_count > 1) {
3189		- // Multiple chunks - show grouped row with expand button
3190		- $group_id = 'group-' . md5($source_url);
3191		- ?>
3192		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
3193		- class="mxchat-chunk-group-header"
3194		- data-source="<?php echo esc_attr($data_source); ?>"
3195		- data-group-id="<?php echo esc_attr($group_id); ?>"
3196		- style="border-bottom: 1px solid var(--mxch-card-border);">
3197		- <td style="padding: 12px 16px; text-align: center;">
3198		- <input type="checkbox"
3199		- class="mxchat-entry-checkbox"
3200		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
3201		- data-source="<?php echo esc_attr($data_source); ?>"
3202		- data-source-url="<?php echo esc_attr($source_url); ?>"
3203		- data-is-group="true"
3204		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
3205		- </td>
3206		- <td style="padding: 12px 16px; font-size: 13px;">
3207		- <?php echo esc_html($first_prompt->id); ?>
3208		- </td>
3209		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3210		- <div class="mxchat-chunk-group-info">
3211		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
3212		- <span class="dashicons dashicons-arrow-right-alt2"></span>
3213		- </button>
3214		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
3215		- <span class="mxchat-chunk-preview">
3216		- <?php
3217		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : $first_prompt->article_content;
3218		- $content_preview = mb_substr($parent_content, 0, 100);
3219		- echo esc_html($content_preview . '...');
3220		- ?>
3221		- </span>
3222		- </div>
3223		- </td>
3224		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3225		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
3226		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
3227		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
3228		- <?php esc_html_e('View Source', 'mxchat'); ?>
3229		- </a>
3230		- <?php else : ?>
3231		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
3232		- <?php endif; ?>
3233		- </td>
3234		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
3235		- <?php if ($data_source !== 'pinecone') : ?>
3236		- <button type="button"
3237		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
3238		- data-source-url="<?php echo esc_attr($source_url); ?>"
3239		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
3240		- data-data-source="<?php echo esc_attr($data_source); ?>"
3241		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3242		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
3243		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
3244		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
3245		- </button>
3246		- <?php endif; ?>
3247		- <button type="button"
3248		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
3249		- data-source-url="<?php echo esc_attr($source_url); ?>"
3250		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
3251		- data-data-source="<?php echo esc_attr($data_source); ?>"
3252		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3253		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
3254		- style="color: var(--mxch-error);"
3255		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
3256		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
3257		- </button>
3258		- </td>
3259		- </tr>
3260		- <?php
3261		- // Render hidden chunk rows
3262		- foreach ($group as $chunk_index => $chunk) {
3263		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
3264		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
3265		- $content = isset($chunk->display_content) ? $chunk->display_content : $chunk->article_content;
3266		- $content_preview = mb_strlen($content) > $preview_length
3267		- ? mb_substr($content, 0, $preview_length) . '...'
3268		- : $content;
3269		- ?>
3270		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
3271		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
3272		- data-source="<?php echo esc_attr($data_source); ?>"
3273		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
3274		- <td style="padding: 12px 16px; text-align: center;">
3275		- <!-- Checkbox column placeholder for chunks (managed by group) -->
3276		- </td>
3277		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
3278		- <!-- Hidden ID column for chunks -->
3279		- </td>
3280		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3281		- <div class="mxchat-accordion-wrapper">
3282		- <div class="mxchat-content-preview">
3283		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
3284		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
3285		- </span>
3286		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
3287		- <?php if (mb_strlen($content) > $preview_length) : ?>
3288		- <button class="mxchat-expand-toggle" type="button">
3289		- <span class="dashicons dashicons-arrow-down-alt2"></span>
3290		- </button>
3291		- <?php endif; ?>
3292		- </div>
3293		- <div class="mxchat-content-full" style="display: none;">
3294		- <div class="content-view">
3295		- <?php
3296		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
3297		- echo '<div dir="rtl" lang="he" class="rtl-content">';
3298		- echo wp_kses_post(wpautop($content));
3299		- echo '</div>';
3300		- } else {
3301		- echo wp_kses_post(wpautop($content));
3302		- }
3303		- ?>
3304		- </div>
3305		- </div>
3306		- </div>
3307		- </td>
3308		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3309		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
3310		- </td>
3311		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
3312		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
3313		- </td>
3314		- </tr>
3315		- <?php
3316		- }
3317		- } else {
3318		- // Single entry - display normally with accordion
3319		- $prompt = $first_prompt;
3320		- $content = isset($prompt->display_content) ? $prompt->display_content : $prompt->article_content;
3321		- $content_preview = mb_strlen($content) > $preview_length
3322		- ? mb_substr($content, 0, $preview_length) . '...'
3323		- : $content;
3324		- ?>
3325		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
3326		- data-source="<?php echo esc_attr($data_source); ?>"
3327		- style="border-bottom: 1px solid var(--mxch-card-border);">
3328		- <td style="padding: 12px 16px; text-align: center;">
3329		- <input type="checkbox"
3330		- class="mxchat-entry-checkbox"
3331		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
3332		- data-source="<?php echo esc_attr($data_source); ?>"
3333		- data-source-url="<?php echo esc_attr($source_url); ?>"
3334		- data-is-group="false">
3335		- </td>
3336		- <td style="padding: 12px 16px; font-size: 13px;">
3337		- <?php echo esc_html($prompt->id); ?>
3338		- </td>
3339		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3340		- <div class="mxchat-accordion-wrapper">
3341		- <div class="mxchat-content-preview">
3342		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
3343		- <?php if (mb_strlen($content) > $preview_length) : ?>
3344		- <button class="mxchat-expand-toggle" type="button">
3345		- <span class="dashicons dashicons-arrow-down-alt2"></span>
3346		- </button>
3347		- <?php endif; ?>
3348		- </div>
3349		- <div class="mxchat-content-full" style="display: none;">
3350		- <div class="content-view">
3351		- <?php
3352		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
3353		- echo '<div dir="rtl" lang="he" class="rtl-content">';
3354		- echo wp_kses_post(wpautop($content));
3355		- echo '</div>';
3356		- } else {
3357		- echo wp_kses_post(wpautop($content));
3358		- }
3359		- ?>
3360		- </div>
3361		- </div>
3362		- </div>
3363		- </td>
3364		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3365		- <?php
3366		- $actual_source = $source_url;
3367		- if (strpos($source_url, '_ungrouped_') === 0) {
3368		- $actual_source = $prompt->source_url ?? '';
3369		- }
3370		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
3371		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
3372		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
3373		- <?php esc_html_e('View', 'mxchat'); ?>
3374		- </a>
3375		- <?php else : ?>
3376		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
3377		- <?php endif; ?>
3378		- </td>
3379		- <td style="padding: 12px 16px; white-space: nowrap;">
3380		- <button type="button"
3381		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
3382		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
3383		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
3384		- data-data-source="<?php echo esc_attr($data_source); ?>"
3385		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3386		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
3387		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
3388		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
3389		- </button>
3390		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-wordpress" data-entry-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_wordpress_prompt_nonce'); ?>" style="color: var(--mxch-error);">
3391		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
3392		- </button>
3393		- </td>
3394		- </tr>
3395		- <?php
3396		- }
3397		- }
3398		- }
3399		- $html = ob_get_clean();
3400		-
3401		- // Generate pagination HTML (include search/filter data for subsequent pages)
3402		- $pagination_html = '';
3403		- if ($total_pages > 1) {
3404		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
3405		-
3406		- // Previous button
3407		- if ($page > 1) {
3408		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
3409		- }
3410		-
3411		- // Page numbers
3412		- $start_page = max(1, $page - 2);
3413		- $end_page = min($total_pages, $page + 2);
3414		-
3415		- if ($start_page > 1) {
3416		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
3417		- if ($start_page > 2) {
3418		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
3419		- }
3420		- }
3421		-
3422		- for ($i = $start_page; $i <= $end_page; $i++) {
3423		- if ($i == $page) {
3424		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
3425		- } else {
3426		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
3427		- }
3428		- }
3429		-
3430		- if ($end_page < $total_pages) {
3431		- if ($end_page < $total_pages - 1) {
3432		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
3433		- }
3434		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
3435		- }
3436		-
3437		- // Next button
3438		- if ($page < $total_pages) {
3439		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
3440		- }
3441		-
3442		- $pagination_html .= '</div>';
3443		- }
3444		-
3445		- wp_send_json_success(array(
3446		- 'html' => $html,
3447		- 'pagination_html' => $pagination_html,
3448		- 'total_count' => $total_records,
3449		- 'total_pages' => $total_pages,
3450		- 'page' => $page,
3451		- 'per_page' => $per_page,
3452		- 'data_source' => 'wordpress'
3453		- ));
3454		-}
3455		-
3456		-/**
3457		- * AJAX handler to detect available sitemaps on the site
3458		- * Optimized for speed - only checks primary sitemap indexes first
3459		- */
3460		-public function ajax_mxchat_detect_sitemaps() {
3461		- check_ajax_referer('mxchat_detect_sitemaps_nonce', 'nonce');
3462		-
3463		- if (!current_user_can('manage_options')) {
3464		- wp_send_json_error(array('message' => 'Unauthorized'));
3465		- return;
3466		- }
3467		-
3468		- $site_url = get_site_url();
3469		- $sitemaps = array();
3470		- $found_index = false;
3471		-
3472		- // Only check the main sitemap index files first (much faster)
3473		- // These are the primary entry points that contain sub-sitemaps
3474		- $primary_indexes = array(
3475		- 'sitemap_index.xml' => 'Yoast SEO / Rank Math', // Yoast & Rank Math
3476		- 'wp-sitemap.xml' => 'WordPress Core', // WordPress Core
3477		- 'sitemap.xml' => 'Standard', // Generic/AIOSEO
3478		- );
3479		-
3480		- foreach ($primary_indexes as $path => $source) {
3481		- $url = trailingslashit($site_url) . $path;
3482		-
3483		- $response = wp_remote_head($url, array(
3484		- 'timeout' => 10,
3485		- 'sslverify' => false,
3486		- 'redirection' => 1,
3487		- 'user-agent' => mxchat_ingest_user_agent(),
3488		- ));
3489		-
3490		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3491		- // Found a sitemap index - parse it to get sub-sitemaps
3492		- $sub_sitemaps = $this->parse_sitemap_index($url);
3493		- if (!empty($sub_sitemaps)) {
3494		- $sitemaps[] = array(
3495		- 'url' => $url,
3496		- 'type' => 'index',
3497		- 'source' => $source,
3498		- 'sub_sitemaps' => $sub_sitemaps
3499		- );
3500		- $found_index = true;
3501		- // Found a valid index, no need to check others
3502		- break;
3503		- }
3504		- }
3505		- }
3506		-
3507		- // If no sitemap index found, check for standalone sitemaps
3508		- if (!$found_index) {
3509		- $standalone_sitemaps = array(
3510		- 'post-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
3511		- 'page-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
3512		- );
3513		-
3514		- foreach ($standalone_sitemaps as $path => $info) {
3515		- $url = trailingslashit($site_url) . $path;
3516		-
3517		- $response = wp_remote_head($url, array(
3518		- 'timeout' => 2,
3519		- 'sslverify' => false
3520		- ));
3521		-
3522		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3523		- $sitemaps[] = array(
3524		- 'url' => $url,
3525		- 'type' => $info['type'],
3526		- 'source' => $info['source'],
3527		- 'url_count' => 0 // Skip URL count for speed
3528		- );
3529		- }
3530		- }
3531		- }
3532		-
3533		- wp_send_json_success(array(
3534		- 'sitemaps' => $sitemaps,
3535		- 'site_url' => $site_url
3536		- ));
3537		-}
3538		-
3539		-/**
3540		- * Parse a sitemap index to get sub-sitemaps
3541		- * Optimized: doesn't fetch URL count for each sub-sitemap (too slow)
3542		- */
3543		-private function parse_sitemap_index($url) {
3544		- $sub_sitemaps = array();
3545		-
3546		- $response = wp_remote_get($url, array(
3547		- 'timeout' => 30,
3548		- 'sslverify' => false,
3549		- 'user-agent' => mxchat_ingest_user_agent(),
3550		- 'headers' => array(
3551		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
3552		- ),
3553		- ));
3554		-
3555		- if (is_wp_error($response)) {
3556		- return $sub_sitemaps;
3557		- }
3558		-
3559		- $body = wp_remote_retrieve_body($response);
3560		- if (empty($body)) {
3561		- return $sub_sitemaps;
3562		- }
3563		-
3564		- // Suppress XML errors
3565		- libxml_use_internal_errors(true);
3566		- $xml = simplexml_load_string($body);
3567		- libxml_clear_errors();
3568		-
3569		- if ($xml === false) {
3570		- return $sub_sitemaps;
3571		- }
3572		-
3573		- // Check if it's a sitemap index (contains <sitemap> elements)
3574		- if (isset($xml->sitemap)) {
3575		- foreach ($xml->sitemap as $sitemap) {
3576		- $loc = (string) $sitemap->loc;
3577		- if (!empty($loc)) {
3578		- // Try to determine the type from the URL
3579		- $type = 'content';
3580		- if (strpos($loc, 'category') !== false || strpos($loc, 'tag') !== false || strpos($loc, 'taxonomy') !== false) {
3581		- $type = 'taxonomy';
3582		- } elseif (strpos($loc, 'author') !== false || strpos($loc, 'user') !== false) {
3583		- $type = 'author';
3584		- }
3585		-
3586		- // Skip URL count - too slow to fetch for each sitemap
3587		- $sub_sitemaps[] = array(
3588		- 'url' => $loc,
3589		- 'type' => $type,
3590		- 'url_count' => 0, // Don't fetch - takes too long
3591		- 'name' => basename(parse_url($loc, PHP_URL_PATH))
3592		- );
3593		- }
3594		- }
3595		- }
3596		-
3597		- return $sub_sitemaps;
3598		-}
3599		-
3600		-/**
3601		- * Get URL count from a sitemap
3602		- */
3603		-private function get_sitemap_url_count($url) {
3604		- $response = wp_remote_get($url, array(
3605		- 'timeout' => 30,
3606		- 'sslverify' => false,
3607		- 'user-agent' => mxchat_ingest_user_agent(),
3608		- 'headers' => array(
3609		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
3610		- ),
3611		- ));
3612		-
3613		- if (is_wp_error($response)) {
3614		- return 0;
3615		- }
3616		-
3617		- $body = wp_remote_retrieve_body($response);
3618		- if (empty($body)) {
3619		- return 0;
3620		- }
3621		-
3622		- // Count <url> or <loc> elements
3623		- $count = preg_match_all('/<url>/i', $body, $matches);
3624		- return $count ?: 0;
3625		-}
3626		-
3627		-/**
3628		- * Get sitemaps declared in robots.txt
3629		- */
3630		-private function get_sitemaps_from_robots($site_url) {
3631		- $sitemaps = array();
3632		- $robots_url = trailingslashit($site_url) . 'robots.txt';
3633		-
3634		- $response = wp_remote_get($robots_url, array(
3635		- 'timeout' => 15,
3636		- 'sslverify' => false,
3637		- 'user-agent' => mxchat_ingest_user_agent(),
3638		- ));
3639		-
3640		- if (is_wp_error($response)) {
3641		- return $sitemaps;
3642		- }
3643		-
3644		- $body = wp_remote_retrieve_body($response);
3645		- if (empty($body)) {
3646		- return $sitemaps;
3647		- }
3648		-
3649		- // Find Sitemap: declarations
3650		- if (preg_match_all('/^Sitemap:\s*(.+)$/mi', $body, $matches)) {
3651		- foreach ($matches[1] as $sitemap_url) {
3652		- $sitemap_url = trim($sitemap_url);
3653		- if (filter_var($sitemap_url, FILTER_VALIDATE_URL)) {
3654		- $sitemaps[] = $sitemap_url;
3655		- }
3656		- }
3657		- }
3658		-
3659		- return $sitemaps;
3660		-}
3661		-
3662	1767	public function mxchat_stop_processing() {
3663	1768	// Verify permissions
3664	1769	if (!current_user_can('manage_options')) {
3665	1770	wp_die(esc_html__('Unauthorized access', 'mxchat'));
		@@ -3667,52 +1772,28 @@
3667	1772
3668	1773	// Verify nonce
3669	1774	check_admin_referer('mxchat_stop_processing_action', 'mxchat_stop_processing_nonce');
3670	1775
3671		- global $wpdb;
3672		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
3673		-
3674		- // Get active queue IDs
3675		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
3676		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
3677		-
3678		- // Delete all pending items from active queues
3679		- if ($sitemap_queue_id) {
3680		- $wpdb->delete(
3681		- $table_name,
3682		- array(
3683		- 'queue_id' => $sitemap_queue_id,
3684		- 'status' => 'pending'
3685		- ),
3686		- array('%s', '%s')
3687		- );
3688		-
3689		- delete_transient('mxchat_active_queue_sitemap');
	1776	+ // Get the last sitemap URL and clear its transient
	1777	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
	1778	+ if ($sitemap_url) {
	1779	+ delete_transient('mxchat_sitemap_status_' . md5($sitemap_url));
3690	1780	delete_transient('mxchat_last_sitemap_url');
3691	1781	}
3692		-
3693		- if ($pdf_queue_id) {
3694		- // Get PDF path before deleting
3695		- $pdf_path = $this->mxchat_get_queue_meta($pdf_queue_id, 'pdf_path');
3696		-
3697		- $wpdb->delete(
3698		- $table_name,
3699		- array(
3700		- 'queue_id' => $pdf_queue_id,
3701		- 'status' => 'pending'
3702		- ),
3703		- array('%s', '%s')
3704		- );
3705		-
3706		- // Delete PDF file
3707		- if ($pdf_path && file_exists($pdf_path)) {
3708		- wp_delete_file($pdf_path);
3709		- }
3710		-
3711		- delete_transient('mxchat_active_queue_pdf');
	1782	+
	1783	+ // Get the last PDF URL and clear its transient
	1784	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1785	+ if ($pdf_url) {
	1786	+ delete_transient('mxchat_pdf_status_' . md5($pdf_url));
3712	1787	delete_transient('mxchat_last_pdf_url');
3713	1788	}
3714	1789
	1790	+ // Unschedule any pending sitemap events
	1791	+ $timestamp = wp_next_scheduled('mxchat_process_sitemap_urls');
	1792	+ if ($timestamp) {
	1793	+ wp_unschedule_event($timestamp, 'mxchat_process_sitemap_urls');
	1794	+ }
	1795	+
3715	1796	// Redirect back with a success message
3716	1797	set_transient('mxchat_admin_notice_success',
3717	1798	esc_html__('Processing has been stopped successfully.', 'mxchat'),
3718	1799	30
		@@ -3719,12 +1800,8 @@
3719	1800	);
3720	1801	wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
3721	1802	exit;
3722	1803	}
3723		-
3724		-/**
3725		- * Get content list for processing
3726		- */
3727	1804	public function ajax_mxchat_get_content_list() {
3728	1805	// Verify the nonce
3729	1806	check_ajax_referer('mxchat_content_selector_nonce', 'nonce');
3730	1807
		@@ -3732,9 +1809,9 @@
3732	1809	wp_send_json_error(__('Unauthorized access', 'mxchat'));
3733	1810	}
3734	1811
3735	1812	$page = isset($_GET['page']) ? absint($_GET['page']) : 1;
3736		- $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 100;
	1813	+ $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 20;
3737	1814	$search = isset($_GET['search']) ? sanitize_text_field($_GET['search']) : '';
3738	1815	$post_type = isset($_GET['post_type']) ? sanitize_text_field($_GET['post_type']) : 'all';
3739	1816	$post_status = isset($_GET['post_status']) ? sanitize_text_field($_GET['post_status']) : 'publish';
3740	1817	$processed_filter = isset($_GET['processed_filter']) ? sanitize_text_field($_GET['processed_filter']) : 'all';
		@@ -3747,47 +1824,20 @@
3747	1824	'orderby' => 'date',
3748	1825	'order' => 'DESC',
3749	1826	);
3750	1827
3751		- // Handle post types - IMPROVED VERSION
	1828	+ // Handle post types
3752	1829	if ($post_type !== 'all') {
3753	1830	$args['post_type'] = $post_type;
3754	1831	} else {
3755		- // Get all available post types that might contain content
3756		- $all_post_types = array();
	1832	+ // Default to post and page if we can't get post types
	1833	+ $args['post_type'] = array('post', 'page');
3757	1834
3758		- // First get all public post types
3759		- $public_types = get_post_types(array('public' => true), 'names');
3760		- $all_post_types = array_merge($all_post_types, $public_types);
3761		-
3762		- // Add common forum/community post types
3763		- $forum_types = array('topic', 'reply', 'forum', 'wpforo_topic', 'wpforo_post');
3764		- foreach ($forum_types as $forum_type) {
3765		- if (post_type_exists($forum_type)) {
3766		- $all_post_types[] = $forum_type;
3767		- }
	1835	+ // Try to get public post types
	1836	+ $public_types = $this->mxchat_get_public_post_types();
	1837	+ if (is_array($public_types) && !empty($public_types)) {
	1838	+ $args['post_type'] = array_keys($public_types);
3768	1839	}
3769		-
3770		- // Add other commonly used post types
3771		- $common_types = array('product', 'job_listing', 'event', 'portfolio');
3772		- foreach ($common_types as $common_type) {
3773		- if (post_type_exists($common_type)) {
3774		- $all_post_types[] = $common_type;
3775		- }
3776		- }
3777		-
3778		- // Remove duplicates and ensure we have at least some post types
3779		- $all_post_types = array_unique($all_post_types);
3780		-
3781		- if (empty($all_post_types)) {
3782		- // Fallback to basic post types
3783		- $all_post_types = array('post', 'page');
3784		- }
3785		-
3786		- $args['post_type'] = $all_post_types;
3787		-
3788		- // Debug logging to see what post types are being queried
3789		- //error_log('MxChat Debug: Querying post types: ' . implode(', ', $all_post_types));
3790	1840	}
3791	1841
3792	1842	if (!empty($search)) {
3793	1843	$args['s'] = $search;
		@@ -3792,58 +1842,36 @@
3792	1842	if (!empty($search)) {
3793	1843	$args['s'] = $search;
3794	1844	}
3795	1845
3796		- // Get processed data from storage
	1846	+ // ================================
	1847	+ // FIXED: Check only the ACTIVE storage method
	1848	+ // ================================
	1849	+
3797	1850	$processed_data = array();
3798	1851
	1852	+ // Check if Pinecone is enabled
3799	1853	$pinecone_options = get_option('mxchat_pinecone_addon_options', array());
3800	1854	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3801		-
	1855	+
3802	1856	if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3803		- // Get fresh data from Pinecone - no caching
	1857	+ // ONLY check Pinecone if it's enabled
3804	1858	$processed_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
3805	1859	} else {
3806		- // WordPress DB checking with better URL matching for all post types
	1860	+ // ONLY check WordPress DB if Pinecone is not enabled
3807	1861	global $wpdb;
3808	1862	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3809		- $processed_items = $wpdb->get_results("SELECT id, source_url, article_content, timestamp FROM {$table_name}");
3810		-
3811		- // Group items by source_url to count chunks
3812		- $url_chunk_counts = array();
3813		- $url_latest_timestamp = array();
3814		- $url_first_id = array();
3815		-
	1863	+ $processed_items = $wpdb->get_results("SELECT id, source_url, timestamp FROM {$table_name}");
	1864	+
3816	1865	if (!empty($processed_items)) {
3817	1866	foreach ($processed_items as $item) {
3818		- $url = $item->source_url;
3819		- if (empty($url)) continue;
3820		-
3821		- // Count chunks per URL
3822		- if (!isset($url_chunk_counts[$url])) {
3823		- $url_chunk_counts[$url] = 0;
3824		- $url_latest_timestamp[$url] = $item->timestamp;
3825		- $url_first_id[$url] = $item->id;
3826		- }
3827		- $url_chunk_counts[$url]++;
3828		-
3829		- // Track latest timestamp
3830		- if (strtotime($item->timestamp) > strtotime($url_latest_timestamp[$url])) {
3831		- $url_latest_timestamp[$url] = $item->timestamp;
3832		- }
3833		- }
3834		-
3835		- // Now build processed_data with chunk counts
3836		- foreach ($url_chunk_counts as $url => $chunk_count) {
3837		- $post_id = $this->mxchat_url_to_post_id_improved($url);
3838		-
	1867	+ $post_id = url_to_postid($item->source_url);
3839	1868	if ($post_id) {
3840	1869	$processed_data[$post_id] = array(
3841		- 'db_id' => $url_first_id[$url],
3842		- 'timestamp' => $url_latest_timestamp[$url],
3843		- 'url' => $url,
3844		- 'source' => 'wordpress',
3845		- 'chunk_count' => $chunk_count
	1870	+ 'db_id' => $item->id,
	1871	+ 'timestamp' => $item->timestamp,
	1872	+ 'url' => $item->source_url,
	1873	+ 'source' => 'wordpress'
3846	1874	);
3847	1875	}
3848	1876	}
3849	1877	}
		@@ -3848,8 +1876,10 @@
3848	1876	}
3849	1877	}
3850	1878	}
3851	1879
	1880	+ // ================================
	1881	+
3852	1882	// Get processed IDs as a simple array for in_array checks
3853	1883	$processed_ids = array_keys($processed_data);
3854	1884
3855	1885	// Handle processed/unprocessed filter
		@@ -3891,14 +1921,8 @@
3891	1921	$db_record_id = $item_data['db_id'];
3892	1922	}
3893	1923	}
3894	1924
3895		- // Get chunk count for this item
3896		- $chunk_count = 0;
3897		- if ($is_processed && isset($processed_data[$id]['chunk_count'])) {
3898		- $chunk_count = intval($processed_data[$id]['chunk_count']);
3899		- }
3900		-
3901	1925	$content_items[] = array(
3902	1926	'id' => $id,
3903	1927	'title' => get_the_title(),
3904	1928	'permalink' => get_permalink(),
		@@ -3909,10 +1933,9 @@
3909	1933	'word_count' => $word_count,
3910	1934	'already_processed' => $is_processed,
3911	1935	'processed_date' => $processed_date,
3912	1936	'db_record_id' => $db_record_id,
3913		- 'data_source' => $data_source,
3914		- 'chunk_count' => $chunk_count
	1937	+ 'data_source' => $data_source
3915	1938	);
3916	1939	}
3917	1940	wp_reset_postdata();
3918	1941	}
		@@ -3928,178 +1951,8 @@
3928	1951	wp_send_json_success($response);
3929	1952	exit;
3930	1953	}
3931	1954
3932		-
3933		-/**
3934		- * This function handles various WooCommerce URL formats and permalink structures
3935		- */
3936		-private function mxchat_url_to_post_id_improved($url) {
3937		- // First try the standard WordPress function
3938		- $post_id = url_to_postid($url);
3939		-
3940		- if ($post_id > 0) {
3941		- return $post_id;
3942		- }
3943		-
3944		- // If that fails, try more aggressive URL matching
3945		- // Remove trailing slashes and query parameters for better matching
3946		- $clean_url = rtrim($url, '/');
3947		- $clean_url = strtok($clean_url, '?'); // Remove query parameters
3948		-
3949		- // Try again with cleaned URL
3950		- $post_id = url_to_postid($clean_url);
3951		- if ($post_id > 0) {
3952		- return $post_id;
3953		- }
3954		-
3955		- // For bbPress forum topics, try extracting slug from URL
3956		- if (strpos($url, '/topic/') !== false || strpos($url, '/forums/') !== false) {
3957		- // Handle bbPress URLs: /forums/topic/topic-name/
3958		- if (preg_match('/\/forums\/topic\/([^\/\?]+)/', $url, $matches)) {
3959		- $topic_slug = $matches[1];
3960		-
3961		- // Look up topic by slug
3962		- $topic = get_page_by_path($topic_slug, OBJECT, 'topic');
3963		- if ($topic) {
3964		- return $topic->ID;
3965		- }
3966		-
3967		- // Alternative method: query by post_name
3968		- global $wpdb;
3969		- $post_id = $wpdb->get_var($wpdb->prepare(
3970		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3971		- $topic_slug
3972		- ));
3973		-
3974		- if ($post_id) {
3975		- return intval($post_id);
3976		- }
3977		- }
3978		-
3979		- // Handle simpler topic URLs: /topic/topic-name/
3980		- if (preg_match('/\/topic\/([^\/\?]+)/', $url, $matches)) {
3981		- $topic_slug = $matches[1];
3982		-
3983		- global $wpdb;
3984		- $post_id = $wpdb->get_var($wpdb->prepare(
3985		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3986		- $topic_slug
3987		- ));
3988		-
3989		- if ($post_id) {
3990		- return intval($post_id);
3991		- }
3992		- }
3993		- }
3994		-
3995		- // For WooCommerce products
3996		- if (strpos($url, '/product/') !== false || strpos($url, 'product=') !== false) {
3997		- // Extract product slug from various URL formats
3998		- $product_slug = '';
3999		-
4000		- // Handle pretty permalinks: /product/product-name/
4001		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
4002		- $product_slug = $matches[1];
4003		- }
4004		- // Handle query parameters: ?product=product-name
4005		- elseif (preg_match('/[\?&]product=([^&]+)/', $url, $matches)) {
4006		- $product_slug = $matches[1];
4007		- }
4008		-
4009		- if (!empty($product_slug)) {
4010		- // Look up product by slug
4011		- $product = get_page_by_path($product_slug, OBJECT, 'product');
4012		- if ($product) {
4013		- return $product->ID;
4014		- }
4015		-
4016		- // Alternative method: query by post_name
4017		- global $wpdb;
4018		- $post_id = $wpdb->get_var($wpdb->prepare(
4019		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'product' AND post_status = 'publish'",
4020		- $product_slug
4021		- ));
4022		-
4023		- if ($post_id) {
4024		- return intval($post_id);
4025		- }
4026		- }
4027		- }
4028		-
4029		- // Generic approach: try to extract slug and match against all post types
4030		- $parsed_url = wp_parse_url($clean_url);
4031		- $path = $parsed_url['path'] ?? '';
4032		-
4033		- if (!empty($path)) {
4034		- // Get the last part of the path as potential slug
4035		- $path_parts = array_filter(explode('/', trim($path, '/')));
4036		- $potential_slug = end($path_parts);
4037		-
4038		- if (!empty($potential_slug)) {
4039		- global $wpdb;
4040		-
4041		- // Try to find any post with this slug
4042		- $post_id = $wpdb->get_var($wpdb->prepare(
4043		- "SELECT ID FROM {$wpdb->posts}
4044		- WHERE post_name = %s
4045		- AND post_status IN ('publish', 'closed', 'private')
4046		- AND post_type NOT IN ('revision', 'attachment', 'nav_menu_item')
4047		- ORDER BY CASE
4048		- WHEN post_type = 'post' THEN 1
4049		- WHEN post_type = 'page' THEN 2
4050		- WHEN post_type = 'topic' THEN 3
4051		- WHEN post_type = 'product' THEN 4
4052		- ELSE 5
4053		- END
4054		- LIMIT 1",
4055		- $potential_slug
4056		- ));
4057		-
4058		- if ($post_id) {
4059		- return intval($post_id);
4060		- }
4061		- }
4062		- }
4063		-
4064		- // ADDITIONAL: Try direct database lookup by URL variations
4065		- global $wpdb;
4066		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
4067		-
4068		- // Try variations of the URL (with/without trailing slash, http/https)
4069		- $url_variations = array(
4070		- $url,
4071		- rtrim($url, '/'),
4072		- $url . '/',
4073		- str_replace('http://', 'https://', $url),
4074		- str_replace('https://', 'http://', $url),
4075		- str_replace('http://', 'https://', rtrim($url, '/')),
4076		- str_replace('https://', 'http://', rtrim($url, '/'))
4077		- );
4078		-
4079		- // Remove duplicates
4080		- $url_variations = array_unique($url_variations);
4081		-
4082		- foreach ($url_variations as $variation) {
4083		- $existing_record = $wpdb->get_row($wpdb->prepare(
4084		- "SELECT id, source_url FROM $table_name WHERE source_url = %s",
4085		- $variation
4086		- ));
4087		-
4088		- if ($existing_record) {
4089		- // Try to get post ID from this stored URL
4090		- $stored_post_id = url_to_postid($existing_record->source_url);
4091		- if ($stored_post_id > 0) {
4092		- return $stored_post_id;
4093		- }
4094		- }
4095		- }
4096		-
4097		- return 0; // No match found
4098		-}
4099		-/**
4100		- * Process selected content via AJAX
4101		- */
4102	1955	public function ajax_mxchat_process_selected_content() {
4103	1956	// Basic request validation
4104	1957	if (!check_ajax_referer('mxchat_content_selector_nonce', 'nonce', false)) {
4105	1958	wp_send_json_error('Invalid nonce');
		@@ -4123,16 +1976,8 @@
4123	1976	wp_send_json_error('No content selected');
4124	1977	exit;
4125	1978	}
4126	1979
4127		- // Get bot_id from request
4128		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
4129		-
4130		- // ACF→PDF extraction is an install-level setting (Knowledge → ACF Fields,
4131		- // plan 11720c). The import modal shows a passive status line pointing
4132		- // there; the old per-batch checkbox and its remembered default are gone.
4133		- $extract_acf_pdfs = get_option('mxchat_acf_pdf_extraction', '0') === '1';
4134		-
4135	1980	// Process only ONE post at a time to avoid request size issues
4136	1981	$post_id = reset($post_ids);
4137	1982	$post = get_post($post_id);
4138	1983
		@@ -4139,236 +1984,54 @@
4139	1984	if (!$post) {
4140	1985	wp_send_json_error('Post not found');
4141	1986	exit;
4142	1987	}
4143		-
4144		- /**
4145		- * Allow developers to modify post data before processing into the knowledge base.
4146		- * Applied on BOTH content-preparation paths (this manual bulk import and the
4147		- * auto-sync path in mxchat_handle_post_update) with the same signature, so a
4148		- * callback registered once covers every indexing route. Purely additive —
4149		- * zero behaviour change when unhooked.
4150		- *
4151		- * @param WP_Post $post The post about to be indexed.
4152		- * @param string $bot_id Bot context for this import.
4153		- */
4154		- $post = apply_filters('mxchat_before_process_post', $post, $bot_id);
4155		- if (!($post instanceof WP_Post)) {
4156		- $post = get_post($post_id); // defend against a bad callback return
	1988	+
	1989	+ // Get minimal content
	1990	+ $content = $post->post_title . "\n\n" . wp_strip_all_tags($post->post_content);
	1991	+ $content = substr($content, 0, 10000); // Limit content size
	1992	+
	1993	+ // Get API key with proper model detection
	1994	+ $options = get_option('mxchat_options');
	1995	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	1996	+
	1997	+ if (strpos($selected_model, 'voyage') === 0) {
	1998	+ $api_key = $options['voyage_api_key'] ?? '';
	1999	+ $provider_name = 'Voyage AI';
	2000	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	2001	+ $api_key = $options['gemini_api_key'] ?? '';
	2002	+ $provider_name = 'Google Gemini';
	2003	+ } else {
	2004	+ $api_key = $options['api_key'] ?? '';
	2005	+ $provider_name = 'OpenAI';
4157	2006	}
4158		-
4159		- // Get content including title, short description (for WooCommerce), and main content
4160		- // Entity decode at output time (single-pass, shared helper) — a stored
4161		- // `&amp;` embeds worse than `&` and gets quoted back to visitors (d2c92e).
4162		- $content = $this->mxchat_decode_entities_for_indexing($post->post_title) . "\n\n";
4163		-
4164		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
4165		- // Strip FIRST, then test: an excerpt that is nothing but shortcodes strips to
4166		- // empty, and testing the raw value emitted a bare "Short Description: " label
4167		- // with no value after it. Matches mxchat_index_published_post.
4168		- // trim() only in the TEST — the emitted value is untouched, so a populated
4169		- // excerpt is byte-identical to before. A whitespace-only excerpt is an empty
4170		- // excerpt and must not produce a labelled line with nothing after it.
4171		- $clean_excerpt = $this->strip_shortcode_tags_preserve_content($post->post_excerpt);
4172		- if (trim($clean_excerpt) !== '') {
4173		- $content .= "Short Description: " . $this->mxchat_decode_entities_for_indexing(wp_strip_all_tags($clean_excerpt)) . "\n\n";
4174		- }
4175		-
4176		- // Add main content - remove shortcode tags but preserve content inside them
4177		- $clean_content = $this->strip_shortcode_tags_preserve_content($post->post_content);
4178		- $content .= $this->mxchat_decode_entities_for_indexing(wp_strip_all_tags($clean_content));
4179		-
4180		- // ADD WOOCOMMERCE PRODUCT DATA (pricing, stock, categories, custom tabs)
4181		- if (get_post_type($post_id) === 'product' && class_exists('WooCommerce')) {
4182		- $product = wc_get_product($post_id);
4183		-
4184		- if ($product) {
4185		- $sku = $product->get_sku();
4186		-
4187		- // Add pricing information (base-currency pinned — see mxchat_product_price_lines)
4188		- $content .= "\n";
4189		- $content .= $this->mxchat_product_price_lines($product);
4190		-
4191		- if (!empty($sku)) {
4192		- $content .= "SKU: " . $sku . "\n";
4193		- }
4194		-
4195		- // Get product categories
4196		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
4197		- if (!empty($categories) && !is_wp_error($categories)) {
4198		- $content .= "Categories: " . implode(', ', $categories) . "\n";
4199		- }
4200		- }
4201		-
4202		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
4203		- $custom_tabs = get_post_meta($post_id, 'yikes_woo_products_tabs', true);
4204		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
4205		- foreach ($custom_tabs as $tab) {
4206		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
4207		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
4208		-
4209		- if (!empty($tab_title) && !empty($tab_content)) {
4210		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
4211		- }
4212		- }
4213		- }
4214		-
4215		- // Also check for reusable/saved tabs applied to this product
4216		- $applied_saved_tabs = get_post_meta($post_id, 'yikes_woo_reusable_products_tabs_applied', true);
4217		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
4218		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
4219		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
4220		- foreach ($applied_saved_tabs as $saved_tab_id) {
4221		- if (isset($saved_tabs[$saved_tab_id])) {
4222		- $tab = $saved_tabs[$saved_tab_id];
4223		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
4224		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
4225		-
4226		- if (!empty($tab_title) && !empty($tab_content)) {
4227		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
4228		- }
4229		- }
4230		- }
4231		- }
4232		- }
4233		- }
4234		-
4235		- // For custom post types like job_listing, include additional fields
4236		- // (verbatim parity with mxchat_index_published_post — a bulk import used to
4237		- // index the body alone, losing location/type/company that auto-sync captured)
4238		- if (get_post_type($post_id) === 'job_listing') {
4239		- // Add job-specific meta if available
4240		- $job_location = get_post_meta($post_id, '_job_location', true);
4241		- if (!empty($job_location)) {
4242		- $content .= "\n\nLocation: " . $job_location;
4243		- }
4244		-
4245		- // Get job type terms
4246		- $job_types = get_the_terms($post_id, 'job_listing_type');
4247		- if (!empty($job_types) && !is_wp_error($job_types)) {
4248		- $types = array();
4249		- foreach ($job_types as $type) {
4250		- $types[] = $type->name;
4251		- }
4252		- $content .= "\n\nJob Type: " . implode(', ', $types);
4253		- }
4254		-
4255		- // Get company name if available
4256		- $company_name = get_post_meta($post_id, '_company_name', true);
4257		- if (!empty($company_name)) {
4258		- $content .= "\n\nCompany: " . $company_name;
4259		- }
4260		- }
4261		-
4262		- // ADD ACF FIELDS SUPPORT
4263		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
4264		- $pdf_extracted_count = 0;
4265		- if (!empty($acf_fields)) {
4266		- $acf_content_parts = array();
4267		- $pdf_attachment_ids = array();
4268		-
4269		- foreach ($acf_fields as $field_name => $field_value) {
4270		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
4271		-
4272		- if (!empty($formatted_value)) {
4273		- // Both separators: a hyphenated ACF name should read as words, and
4274		- // this is what mxchat_index_published_post already does.
4275		- $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name));
4276		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
4277		- }
4278		-
4279		- // Walk this field's value tree for any PDF attachment references and queue them for extraction.
4280		- // Only when the user opted into ACF→PDF extraction for this batch; otherwise the ACF text
4281		- // still lands in the KB but the heavier PDF parsing is skipped.
4282		- if ($extract_acf_pdfs) {
4283		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
4284		- }
4285		- }
4286		-
4287		- if (!empty($acf_content_parts)) {
4288		- $content .= "\n\n" . implode("\n", $acf_content_parts);
4289		- }
4290		-
4291		- // Extract text from each unique PDF found in ACF fields and append as a labeled section
4292		- if ($extract_acf_pdfs && !empty($pdf_attachment_ids)) {
4293		- $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
4294		- $pdf_sections = array();
4295		- foreach ($pdf_attachment_ids as $att_id) {
4296		- $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
4297		- if (!empty($pdf_text)) {
4298		- $pdf_title = get_the_title($att_id);
4299		- $pdf_url = wp_get_attachment_url($att_id);
4300		- $header = 'PDF Attachment';
4301		- if (!empty($pdf_title)) {
4302		- $header .= ': ' . $pdf_title;
4303		- }
4304		- if (!empty($pdf_url)) {
4305		- $header .= ' (' . $pdf_url . ')';
4306		- }
4307		- $pdf_sections[] = $header . "\n" . $pdf_text;
4308		- $pdf_extracted_count++;
4309		- }
4310		- }
4311		- if (!empty($pdf_sections)) {
4312		- $content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
4313		- }
4314		- }
4315		- }
4316		-
4317		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
4318		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
4319		- if (!empty($custom_meta)) {
4320		- $meta_content_parts = array();
4321		-
4322		- foreach ($custom_meta as $meta_key => $meta_value) {
4323		- // Convert meta key to readable label
4324		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
4325		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
4326		- }
4327		-
4328		- if (!empty($meta_content_parts)) {
4329		- $content .= "\n\n" . implode("\n", $meta_content_parts);
4330		- }
4331		- }
4332		-
4333		- // Debug logging for WordPress Import content
4334		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Post ID: ' . $post_id . ' Title: ' . $post->post_title);
4335		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Raw post_content length: ' . strlen($post->post_content));
4336		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Final content length: ' . strlen($content));
4337		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Content preview: ' . substr($content, 0, 300));
4338		-
4339		- // Note: Removed 10,000 char limit - chunking now handles large content properly
4340		-
4341		- // Get bot-specific embedding decision — custom-provider-aware (plan cbd5fd)
4342		- $bot_options = $this->get_bot_options($bot_id);
4343		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
4344		-
4345		- $preflight = MxChat_Utils::embedding_preflight($options);
4346		- if (!$preflight['ok']) {
4347		- MxChat_Admin::mxchat_log_debug('api_error', $preflight['reason'] . ' (knowledge processing)');
4348		- wp_send_json_error($preflight['reason']);
	2007	+
	2008	+ if (empty($api_key)) {
	2009	+ wp_send_json_error($provider_name . ' API key not configured');
4349	2010	exit;
4350	2011	}
4351		- $api_key = $preflight['api_key'];
4352	2012
4353	2013	$source_url = get_permalink($post_id);
4354	2014	$vector_id = md5($source_url); // Vector ID for Pinecone
4355	2015
4356		- // Check for existing content in bot-specific storage
	2016	+ // ================================
	2017	+ // FIXED: Check for existing content in ONLY the active storage method
	2018	+ // ================================
	2019	+
4357	2020	$is_update = false;
4358	2021
4359		- // Get bot-specific Pinecone configuration
4360		- $bot_pinecone_config = $this->get_bot_pinecone_config($bot_id);
4361		- $use_pinecone = !empty($bot_pinecone_config) && ($bot_pinecone_config['use_pinecone'] ?? false);
	2022	+ // Check if Pinecone is enabled
	2023	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	2024	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
4362	2025
4363		- if ($use_pinecone && !empty($bot_pinecone_config['api_key'])) {
4364		- // Check Pinecone for this bot
4365		- $pinecone_data = $this->mxchat_get_pinecone_processed_content($bot_pinecone_config);
	2026	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2027	+ // ONLY check Pinecone if it's enabled
	2028	+ $pinecone_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
4366	2029	if (isset($pinecone_data[$post_id])) {
4367	2030	$is_update = true;
4368	2031	}
4369	2032	} else {
4370		- // Check WordPress DB (same as before since it's shared)
	2033	+ // ONLY check WordPress DB if Pinecone is not enabled
4371	2034	global $wpdb;
4372	2035	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
4373	2036	$existing_record = $wpdb->get_row($wpdb->prepare(
4374	2037	"SELECT id FROM $table_name WHERE source_url = %s",
		@@ -4378,54 +2041,54 @@
4378	2041	if ($existing_record) {
4379	2042	$is_update = true;
4380	2043	}
4381	2044	}
4382		-
4383		- // UPDATED 2.5.6: Determine content type based on post_type
4384		- $post_type = $post->post_type;
4385		- $content_type = 'content'; // Default fallback
4386		-
4387		- // Map WordPress post types to content types
4388		- switch ($post_type) {
4389		- case 'post':
4390		- $content_type = 'post';
4391		- break;
4392		- case 'page':
4393		- $content_type = 'page';
4394		- break;
4395		- case 'product':
4396		- $content_type = 'product';
4397		- break;
4398		- default:
4399		- // For custom post types, use the post type name
4400		- $content_type = sanitize_key($post_type);
4401		- break;
4402		- }
4403		-
4404		- // Use the centralized utility function with bot_id and content_type
	2045	+
	2046	+ // Use the centralized utility function for storage
4405	2047	$result = MxChat_Utils::submit_content_to_db(
4406		- $content,
4407		- $source_url,
	2048	+ $content,
	2049	+ $source_url,
4408	2050	$api_key,
4409		- $vector_id,
4410		- $bot_id,
4411		- $content_type
	2051	+ $vector_id
4412	2052	);
4413		-
	2053	+
4414	2054	if (is_wp_error($result)) {
4415		- MxChat_Admin::mxchat_log_debug('storage_error', 'Knowledge storage failed: ' . $result->get_error_message(), array('source_url' => $source_url));
4416	2055	wp_send_json_error('Storage failed: ' . $result->get_error_message());
4417	2056	exit;
4418	2057	}
4419		-
4420		- // Automatically apply role restriction based on tags
4421		- $this->apply_role_restriction_to_post($post_id, $source_url);
4422		-
	2058	+
	2059	+ // ================================
	2060	+ // UPDATE: Only update caches if Pinecone is enabled
	2061	+ // ================================
	2062	+
	2063	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2064	+ // Update vector ID cache for improved fetching
	2065	+ $this->mxchat_update_pinecone_vector_cache($vector_id);
	2066	+
	2067	+ // Update local processed content cache for immediate UI feedback
	2068	+ $pinecone_cache = get_option('mxchat_pinecone_processed_cache', array());
	2069	+ $pinecone_cache[$post_id] = array(
	2070	+ 'db_id' => $vector_id,
	2071	+ 'processed_date' => 'Just now',
	2072	+ 'url' => $source_url,
	2073	+ 'source' => 'pinecone',
	2074	+ 'timestamp' => current_time('timestamp')
	2075	+ );
	2076	+ update_option('mxchat_pinecone_processed_cache', $pinecone_cache);
	2077	+
	2078	+ // Also update the general processed content cache
	2079	+ $processed_cache = get_option('mxchat_processed_content_cache', array());
	2080	+ $processed_cache[$post_id] = array(
	2081	+ 'db_id' => $vector_id,
	2082	+ 'timestamp' => current_time('timestamp'),
	2083	+ 'url' => $source_url,
	2084	+ 'source' => 'pinecone'
	2085	+ );
	2086	+ update_option('mxchat_processed_content_cache', $processed_cache);
	2087	+ }
	2088	+
4423	2089	$operation_type = $is_update ? 'update' : 'new';
4424	2090
4425		- // Count ACF fields for debugging
4426		- $acf_field_count = count($acf_fields);
4427		-
4428	2091	// Success response with minimal data
4429	2092	wp_send_json_success(array(
4430	2093	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
4431	2094	'post_id' => $post_id,
		@@ -4430,95 +2093,27 @@
4430	2093	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
4431	2094	'post_id' => $post_id,
4432	2095	'title' => $post->post_title,
4433	2096	'operation_type' => $operation_type,
4434		- 'vector_id' => $vector_id,
4435		- 'acf_fields_found' => $acf_field_count,
4436		- 'pdf_extracted_count' => (int) $pdf_extracted_count,
4437		- 'content_preview' => substr($content, 0, 100) . '...',
4438		- 'bot_id' => $bot_id
	2097	+ 'vector_id' => $vector_id, // Include vector ID for debugging
	2098	+ 'cache_updated' => $use_pinecone // Indicate if cache was updated
4439	2099	));
4440	2100	exit;
4441	2101	}
4442	2102
4443		-private function apply_role_restriction_to_post($post_id, $source_url) {
4444		- // Get tag-role mappings
4445		- $mappings = get_option('mxchat_tag_role_mappings', array());
4446		-
4447		- if (empty($mappings)) {
4448		- return; // No mappings, leave as public
4449		- }
4450		-
4451		- // Get all tags for the post
4452		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
4453		-
4454		- if (empty($post_tags)) {
4455		- return; // No tags, leave as public
4456		- }
4457		-
4458		- // Determine the highest role restriction based on tags
4459		- $highest_role = 'public';
4460		- $role_hierarchy = array(
4461		- 'public' => 0,
4462		- 'logged_in' => 1,
4463		- 'subscriber' => 2,
4464		- 'contributor' => 3,
4465		- 'author' => 4,
4466		- 'editor' => 5,
4467		- 'administrator' => 6
4468		- );
4469		-
4470		- foreach ($post_tags as $tag_slug) {
4471		- if (isset($mappings[$tag_slug])) {
4472		- $role = $mappings[$tag_slug];
4473		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
4474		- $highest_role = $role;
4475		- }
4476		- }
4477		- }
4478		-
4479		- // If no restricted tags found, return (leave as public)
4480		- if ($highest_role === 'public') {
4481		- return;
4482		- }
4483		-
4484		- // Update the role restriction in the database
4485		- global $wpdb;
4486		-
4487		- // Check if using Pinecone
4488		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
4489		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
4490		-
4491		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
4492		- // Update Pinecone role restriction
4493		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
4494		- $vector_id = md5($source_url);
4495		-
4496		- $wpdb->replace(
4497		- $roles_table,
4498		- array(
4499		- 'vector_id' => $vector_id,
4500		- 'role_restriction' => $highest_role,
4501		- 'updated_at' => current_time('mysql')
4502		- ),
4503		- array('%s', '%s', '%s')
4504		- );
4505		- } else {
4506		- // Update WordPress DB
4507		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
4508		-
4509		- $wpdb->update(
4510		- $table_name,
4511		- array('role_restriction' => $highest_role),
4512		- array('source_url' => $source_url),
4513		- array('%s'),
4514		- array('%s')
4515		- );
4516		- }
4517		-}
4518	2103
	2104	+
	2105	+ /**
	2106	+ * Updates cache with new vector ID if absent
	2107	+ */
	2108	+ public function mxchat_update_pinecone_vector_cache($vector_id) {
	2109	+ $cached_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2110	+ if (!in_array($vector_id, $cached_ids)) {
	2111	+ $cached_ids[] = $vector_id;
	2112	+ update_option('mxchat_pinecone_vector_ids_cache', $cached_ids);
	2113	+ }
	2114	+ }
4519	2115	public function mxchat_get_public_post_types() {
4520		- // Get all public post types
4521	2116	$post_types = get_post_types(array('public' => true), 'objects');
4522	2117	$post_type_options = array();
4523	2118
4524	2119	foreach ($post_types as $post_type) {
		@@ -4524,47 +2119,41 @@
4524	2119	foreach ($post_types as $post_type) {
4525	2120	$post_type_options[$post_type->name] = $post_type->label;
4526	2121	}
4527	2122
4528		- // Also include common forum/community post types that might not be marked as public
4529		- $additional_types = array(
4530		- 'topic' => 'Forum Topics (bbPress)',
4531		- 'reply' => 'Forum Replies (bbPress)',
4532		- 'forum' => 'Forums (bbPress)',
4533		- 'wpforo_topic' => 'wpForo Topics',
4534		- 'wpforo_post' => 'wpForo Posts'
4535		- );
4536		-
4537		- foreach ($additional_types as $type_name => $type_label) {
4538		- if (post_type_exists($type_name) && !isset($post_type_options[$type_name])) {
4539		- $post_type_options[$type_name] = $type_label;
4540		- }
4541		- }
4542		-
4543	2123	return $post_type_options;
4544	2124	}
4545		-
4546		-/**
4547		- * Retrieves processed content from Pinecone API
4548		- */
4549	2125	public function mxchat_get_pinecone_processed_content($pinecone_options) {
	2126	+ // First check local cache for immediate updates
	2127	+ $cached_data = get_option('mxchat_pinecone_processed_cache', array());
	2128	+
4550	2129	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4551	2130	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
4552		-
	2131	+
4553	2132	if (empty($api_key) || empty($host)) {
4554		- return array();
	2133	+ // Return only cached data if API credentials are missing
	2134	+ return $cached_data;
4555	2135	}
4556		-
	2136	+
4557	2137	$pinecone_data = array();
4558		-
	2138	+
4559	2139	try {
4560		- // Always get fresh data from Pinecone
4561		- $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
4562		-
4563		- // Method 2: Final fallback - try stats endpoint (if available)
	2140	+ // Method 1: Try to get vectors using cached vector IDs first
	2141	+ $cached_vector_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2142	+
	2143	+ if (!empty($cached_vector_ids)) {
	2144	+ $pinecone_data = $this->mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $cached_vector_ids);
	2145	+ }
	2146	+
	2147	+ // Method 2: If no cached IDs or fetch failed, use scanning approach
4564	2148	if (empty($pinecone_data)) {
	2149	+ $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
	2150	+ }
	2151	+
	2152	+ // Method 3: Final fallback - try stats endpoint (if available)
	2153	+ if (empty($pinecone_data)) {
4565	2154	$stats_url = "https://{$host}/describe_index_stats";
4566		-
	2155	+
4567	2156	$response = wp_remote_post($stats_url, array(
4568	2157	'headers' => array(
4569	2158	'Api-Key' => $api_key,
4570	2159	'Content-Type' => 'application/json'
		@@ -4571,42 +2160,59 @@
4571	2160	),
4572	2161	'body' => json_encode(array()),
4573	2162	'timeout' => 30
4574	2163	));
4575		-
	2164	+
4576	2165	if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
4577	2166	$body = wp_remote_retrieve_body($response);
4578	2167	$stats_data = json_decode($body, true);
	2168	+
	2169	+ // Log stats for debugging but don't rely on them for vector listing
	2170	+ //error_log('Pinecone index stats: ' . print_r($stats_data, true));
4579	2171	}
4580	2172	}
4581		-
	2173	+
4582	2174	} catch (Exception $e) {
4583		- // Log error but return fresh data only
	2175	+ //error_log('Pinecone processed content exception: ' . $e->getMessage());
4584	2176	}
4585		-
4586		- return $pinecone_data;
	2177	+
	2178	+ // Merge cached data with Pinecone data
	2179	+ // Cache takes priority for recent updates (within last 5 minutes)
	2180	+ $merged_data = $pinecone_data;
	2181	+
	2182	+ foreach ($cached_data as $post_id => $cache_item) {
	2183	+ $cache_timestamp = $cache_item['timestamp'] ?? 0;
	2184	+ $time_diff = current_time('timestamp') - $cache_timestamp;
	2185	+
	2186	+ // If cache item is recent (less than 5 minutes), prioritize it
	2187	+ if ($time_diff < 300) { // 5 minutes = 300 seconds
	2188	+ $merged_data[$post_id] = $cache_item;
	2189	+ } else {
	2190	+ // If not in Pinecone data and cache is old, keep cache but mark as potentially stale
	2191	+ if (!isset($merged_data[$post_id])) {
	2192	+ $merged_data[$post_id] = $cache_item;
	2193	+ }
	2194	+ }
	2195	+ }
	2196	+
	2197	+ return $merged_data;
4587	2198	}
4588	2199	public function mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $vector_ids) {
4589		- //error_log('=== DEBUG: Starting mxchat_fetch_pinecone_vectors_by_ids ===');
4590		-
4591	2200	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4592	2201	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
4593		-
	2202	+
4594	2203	if (empty($api_key) || empty($host) || empty($vector_ids)) {
4595		- //error_log('DEBUG: Missing parameters for fetch by IDs');
4596	2204	return array();
4597	2205	}
4598		-
	2206	+
4599	2207	try {
4600	2208	$fetch_url = "https://{$host}/vectors/fetch";
4601		- //error_log('DEBUG: Fetch URL: ' . $fetch_url);
4602		- //error_log('DEBUG: Fetching ' . count($vector_ids) . ' vector IDs');
4603		-
	2209	+
4604	2210	// Pinecone fetch API allows fetching specific vectors by ID
4605	2211	$fetch_data = array(
4606	2212	'ids' => array_values($vector_ids)
4607	2213	);
4608		-
	2214	+
4609	2215	$response = wp_remote_post($fetch_url, array(
4610	2216	'headers' => array(
4611	2217	'Api-Key' => $api_key,
4612	2218	'Content-Type' => 'application/json'
		@@ -4613,45 +2219,39 @@
4613	2219	),
4614	2220	'body' => json_encode($fetch_data),
4615	2221	'timeout' => 30
4616	2222	));
4617		-
	2223	+
4618	2224	if (is_wp_error($response)) {
4619		- //error_log('DEBUG: Fetch by IDs WP error: ' . $response->get_error_message());
	2225	+ //error_log('Pinecone fetch by IDs error: ' . $response->get_error_message());
4620	2226	return array();
4621	2227	}
4622		-
	2228	+
4623	2229	$response_code = wp_remote_retrieve_response_code($response);
4624		- //error_log('DEBUG: Fetch response code: ' . $response_code);
4625		-
4626	2230	if ($response_code !== 200) {
4627		- $error_body = wp_remote_retrieve_body($response);
4628		- //error_log('DEBUG: Fetch failed with body: ' . $error_body);
	2231	+ //error_log('Pinecone fetch by IDs failed with code: ' . $response_code);
4629	2232	return array();
4630	2233	}
4631		-
	2234	+
4632	2235	$body = wp_remote_retrieve_body($response);
4633	2236	$data = json_decode($body, true);
4634	2237
4635		- //error_log('DEBUG: Fetch response structure: ' . print_r(array_keys($data), true));
4636		-
4637	2238	if (!isset($data['vectors'])) {
4638		- //error_log('DEBUG: No vectors key in response');
4639	2239	return array();
4640	2240	}
4641		-
	2241	+
4642	2242	$processed_data = array();
4643		-
	2243	+
4644	2244	foreach ($data['vectors'] as $vector_id => $vector_data) {
4645	2245	$metadata = $vector_data['metadata'] ?? array();
4646	2246	$source_url = $metadata['source_url'] ?? '';
4647		-
	2247	+
4648	2248	if (!empty($source_url)) {
4649	2249	$post_id = url_to_postid($source_url);
4650	2250	if ($post_id) {
4651	2251	$created_at = $metadata['created_at'] ?? '';
4652		- $processed_date = 'Recently';
4653		-
	2252	+ $processed_date = 'Recently'; // Default
	2253	+
4654	2254	if (!empty($created_at)) {
4655	2255	$timestamp = is_numeric($created_at) ? $created_at : strtotime($created_at);
4656	2256	if ($timestamp) {
4657	2257	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
		@@ -4656,9 +2256,9 @@
4656	2256	if ($timestamp) {
4657	2257	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
4658	2258	}
4659	2259	}
4660		-
	2260	+
4661	2261	$processed_data[$post_id] = array(
4662	2262	'db_id' => $vector_id,
4663	2263	'processed_date' => $processed_date,
4664	2264	'url' => $source_url,
		@@ -4667,90 +2267,52 @@
4667	2267	);
4668	2268	}
4669	2269	}
4670	2270	}
4671		-
4672		- //error_log('DEBUG: Processed ' . count($processed_data) . ' vectors from fetch');
	2271	+
4673	2272	return $processed_data;
4674		-
	2273	+
4675	2274	} catch (Exception $e) {
4676		- //error_log('DEBUG: Exception in fetch_pinecone_vectors_by_ids: ' . $e->getMessage());
	2275	+ //error_log('Pinecone fetch by IDs exception: ' . $e->getMessage());
4677	2276	return array();
4678	2277	}
4679	2278	}
4680		-
4681		-/**
4682		- * Get embedding dimensions based on the selected model.
4683		- */
4684		-private function mxchat_get_embedding_dimensions() {
4685		- $options = get_option('mxchat_options', array());
4686		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4687		-
4688		- $model_dimensions = array(
4689		- 'text-embedding-ada-002' => 1536,
4690		- 'text-embedding-3-small' => 1536,
4691		- 'text-embedding-3-large' => 3072,
4692		- 'voyage-2' => 1024,
4693		- 'voyage-large-2' => 1536,
4694		- 'voyage-3-large' => 2048,
4695		- 'gemini-embedding-001' => 1536,
4696		- );
4697		-
4698		- if (strpos($selected_model, 'voyage-3-large') === 0) {
4699		- $custom_dimensions = $options['voyage_output_dimension'] ?? 2048;
4700		- return intval($custom_dimensions);
4701		- }
4702		-
4703		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4704		- $custom_dimensions = $options['gemini_output_dimension'] ?? 1536;
4705		- return intval($custom_dimensions);
4706		- }
4707		-
4708		- return $model_dimensions[$selected_model] ?? 1536;
4709		-}
4710		-
4711		-/**
4712		- * Scan Pinecone for processed content
4713		- */
4714	2279	public function mxchat_scan_pinecone_for_processed_content($pinecone_options) {
4715	2280	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4716	2281	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
4717		-
	2282	+
4718	2283	if (empty($api_key) || empty($host)) {
4719	2284	return array();
4720	2285	}
4721		-
	2286	+
4722	2287	try {
4723	2288	// Use multiple random vectors to get better coverage
4724	2289	$all_matches = array();
4725	2290	$seen_ids = array();
4726		-
4727		- // Get correct dimensions for the configured embedding model
4728		- $dimensions = $this->mxchat_get_embedding_dimensions();
4729		-
	2291	+
4730	2292	// Try 3 different random vectors to get better coverage
4731	2293	for ($i = 0; $i < 3; $i++) {
4732	2294	$query_url = "https://{$host}/query";
4733		-
	2295	+
4734	2296	// Generate a random unit vector instead of zeros
4735	2297	$random_vector = array();
4736		- for ($j = 0; $j < $dimensions; $j++) {
4737		- $random_vector[] = (rand(-1000, 1000) / 1000.0);
	2298	+ for ($j = 0; $j < 1536; $j++) {
	2299	+ $random_vector[] = (rand(-1000, 1000) / 1000.0); // Random values between -1 and 1
4738	2300	}
4739		-
	2301	+
4740	2302	// Normalize the vector to unit length
4741	2303	$magnitude = sqrt(array_sum(array_map(function($x) { return $x * $x; }, $random_vector)));
4742	2304	if ($magnitude > 0) {
4743	2305	$random_vector = array_map(function($x) use ($magnitude) { return $x / $magnitude; }, $random_vector);
4744	2306	}
4745		-
	2307	+
4746	2308	$query_data = array(
4747	2309	'includeMetadata' => true,
4748	2310	'includeValues' => false,
4749		- 'topK' => 10000,
	2311	+ 'topK' => 10000, // Get many results
4750	2312	'vector' => $random_vector
4751	2313	);
4752		-
	2314	+
4753	2315	$response = wp_remote_post($query_url, array(
4754	2316	'headers' => array(
4755	2317	'Api-Key' => $api_key,
4756	2318	'Content-Type' => 'application/json'
		@@ -4757,22 +2319,21 @@
4757	2319	),
4758	2320	'body' => json_encode($query_data),
4759	2321	'timeout' => 30
4760	2322	));
4761		-
	2323	+
4762	2324	if (is_wp_error($response)) {
4763	2325	continue;
4764	2326	}
4765		-
	2327	+
4766	2328	$response_code = wp_remote_retrieve_response_code($response);
4767		-
4768	2329	if ($response_code !== 200) {
4769	2330	continue;
4770	2331	}
4771		-
	2332	+
4772	2333	$body = wp_remote_retrieve_body($response);
4773	2334	$data = json_decode($body, true);
4774		-
	2335	+
4775	2336	if (isset($data['matches'])) {
4776	2337	foreach ($data['matches'] as $match) {
4777	2338	$match_id = $match['id'] ?? '';
4778	2339	if (!empty($match_id) && !isset($seen_ids[$match_id])) {
		@@ -4781,30 +2342,24 @@
4781	2342	}
4782	2343	}
4783	2344	}
4784	2345	}
4785		-
4786		- // Convert matches to processed data format, grouping by URL to count chunks
	2346	+
	2347	+ // Convert matches to processed data format
4787	2348	$processed_data = array();
4788		- $url_chunk_counts = array();
4789		-
	2349	+ $vector_ids_for_cache = array();
	2350	+
4790	2351	foreach ($all_matches as $match) {
4791	2352	$metadata = $match['metadata'] ?? array();
4792	2353	$source_url = $metadata['source_url'] ?? '';
4793	2354	$match_id = $match['id'] ?? '';
4794		-
	2355	+
4795	2356	if (!empty($source_url) && !empty($match_id)) {
4796	2357	$post_id = url_to_postid($source_url);
4797	2358	if ($post_id) {
4798		- // Count chunks per post_id
4799		- if (!isset($url_chunk_counts[$post_id])) {
4800		- $url_chunk_counts[$post_id] = 0;
4801		- }
4802		- $url_chunk_counts[$post_id]++;
4803		-
4804	2359	$created_at = $metadata['created_at'] ?? '';
4805		- $processed_date = 'Recently';
4806		-
	2360	+ $processed_date = 'Recently'; // Default
	2361	+
4807	2362	if (!empty($created_at)) {
4808	2363	$timestamp = is_numeric($created_at) ? $created_at : strtotime($created_at);
4809	2364	if ($timestamp) {
4810	2365	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
		@@ -4809,333 +2364,242 @@
4809	2364	if ($timestamp) {
4810	2365	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
4811	2366	}
4812	2367	}
4813		-
4814		- // Only store if not already set, or update with newer timestamp
4815		- if (!isset($processed_data[$post_id]) ||
4816		- ($timestamp ?? 0) > ($processed_data[$post_id]['timestamp'] ?? 0)) {
4817		- $processed_data[$post_id] = array(
4818		- 'db_id' => $match_id,
4819		- 'processed_date' => $processed_date,
4820		- 'url' => $source_url,
4821		- 'source' => 'pinecone',
4822		- 'timestamp' => $timestamp ?? current_time('timestamp')
4823		- );
4824		- }
	2368	+
	2369	+ $processed_data[$post_id] = array(
	2370	+ 'db_id' => $match_id,
	2371	+ 'processed_date' => $processed_date,
	2372	+ 'url' => $source_url,
	2373	+ 'source' => 'pinecone',
	2374	+ 'timestamp' => $timestamp ?? current_time('timestamp')
	2375	+ );
	2376	+
	2377	+ $vector_ids_for_cache[] = $match_id;
4825	2378	}
4826	2379	}
4827	2380	}
4828		-
4829		- // Add chunk counts to processed data
4830		- foreach ($url_chunk_counts as $post_id => $chunk_count) {
4831		- if (isset($processed_data[$post_id])) {
4832		- $processed_data[$post_id]['chunk_count'] = $chunk_count;
4833		- }
	2381	+
	2382	+ // Update the vector IDs cache for future use
	2383	+ if (!empty($vector_ids_for_cache)) {
	2384	+ update_option('mxchat_pinecone_vector_ids_cache', $vector_ids_for_cache);
4834	2385	}
4835		-
	2386	+
4836	2387	return $processed_data;
4837		-
	2388	+
4838	2389	} catch (Exception $e) {
	2390	+ //error_log('Pinecone scan exception: ' . $e->getMessage());
4839	2391	return array();
4840	2392	}
4841		-}
4842		-/**
4843		- * Generate embeddings from input text for MXChat with bot support
4844		- */
4845		-private function mxchat_generate_embedding($text, $bot_id = 'default') {
4846		- // Enable detailed logging for debugging
4847		- //error_log('[MXCHAT-EMBED] Starting embedding generation for bot: ' . $bot_id . '. Text length: ' . strlen($text) . ' bytes');
4848		- //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
	2393	+}
4849	2394
4850		- // Get bot-specific options
4851		- $bot_options = $this->get_bot_options($bot_id);
4852		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	2395	+ /**
	2396	+ * Generates embeddings from input text for MXChat
	2397	+ */
	2398	+ private function mxchat_generate_embedding($text) {
	2399	+ // Enable detailed logging for debugging
	2400	+ //error_log('[MXCHAT-EMBED] Starting embedding generation. Text length: ' . strlen($text) . ' bytes');
	2401	+ //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4853	2402
4854		- // Opt-in: when the custom provider is selected for embeddings, index through
4855		- // the same custom endpoint the query path uses so stored vectors and query
4856		- // vectors share a model. Returns the vector array on success, or an error
4857		- // string on failure (this function's existing failure contract).
4858		- if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
4859		- if (!class_exists('MxChat_Utils')) {
4860		- require_once dirname(__FILE__) . '/../includes/class-mxchat-utils.php';
4861		- }
4862		- return MxChat_Utils::generate_embedding_custom($text, $options);
4863		- }
	2403	+ $options = get_option('mxchat_options');
	2404	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	2405	+ //error_log('[MXCHAT-EMBED] Selected embedding model: ' . $selected_model);
4864	2406
4865		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4866		- //error_log('[MXCHAT-EMBED] Selected embedding model for bot ' . $bot_id . ': ' . $selected_model);
	2407	+ // Determine provider and endpoint
	2408	+ if (strpos($selected_model, 'voyage') === 0) {
	2409	+ $api_key = $options['voyage_api_key'] ?? '';
	2410	+ $endpoint = 'https://api.voyageai.com/v1/embeddings';
	2411	+ $provider_name = 'Voyage AI';
	2412	+ //error_log('[MXCHAT-EMBED] Using Voyage AI API');
	2413	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	2414	+ $api_key = $options['gemini_api_key'] ?? '';
	2415	+ $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
	2416	+ $provider_name = 'Google Gemini';
	2417	+ //error_log('[MXCHAT-EMBED] Using Google Gemini API');
	2418	+ } else {
	2419	+ $api_key = $options['api_key'] ?? '';
	2420	+ $endpoint = 'https://api.openai.com/v1/embeddings';
	2421	+ $provider_name = 'OpenAI';
	2422	+ //error_log('[MXCHAT-EMBED] Using OpenAI API');
	2423	+ }
4867	2424
4868		- // Determine provider and endpoint
4869		- if (strpos($selected_model, 'voyage') === 0) {
4870		- $api_key = $options['voyage_api_key'] ?? '';
4871		- $endpoint = 'https://api.voyageai.com/v1/embeddings';
4872		- $provider_name = 'Voyage AI';
4873		- //error_log('[MXCHAT-EMBED] Using Voyage AI API for bot ' . $bot_id);
4874		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
4875		- $api_key = $options['gemini_api_key'] ?? '';
4876		- $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
4877		- $provider_name = 'Google Gemini';
4878		- //error_log('[MXCHAT-EMBED] Using Google Gemini API for bot ' . $bot_id);
4879		- } else {
4880		- $api_key = $options['api_key'] ?? '';
4881		- $endpoint = 'https://api.openai.com/v1/embeddings';
4882		- $provider_name = 'OpenAI';
4883		- //error_log('[MXCHAT-EMBED] Using OpenAI API for bot ' . $bot_id);
4884		- }
	2425	+ //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
4885	2426
4886		- //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
	2427	+ if (empty($api_key)) {
	2428	+ $error_message = sprintf('Missing %s API key. Please configure your API key in the MxChat settings.', $provider_name);
	2429	+ //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
	2430	+ return $error_message;
	2431	+ }
4887	2432
4888		- if (empty($api_key)) {
4889		- $error_message = sprintf('Missing %s API key for bot %s. Please configure your API key in the bot settings.', $provider_name, $bot_id);
4890		- //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
4891		- return $error_message;
4892		- }
	2433	+ // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
	2434	+ $estimated_tokens = ceil(str_word_count($text) / 0.75);
	2435	+ //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
4893	2436
4894		- // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
4895		- $estimated_tokens = ceil(str_word_count($text) / 0.75);
4896		- //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
	2437	+ if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
	2438	+ //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
	2439	+ // Consider truncating text here
	2440	+ }
4897	2441
4898		- if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
4899		- //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
4900		- // Consider truncating text here
4901		- }
	2442	+ // Prepare request body based on provider
	2443	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2444	+ // Gemini API format
	2445	+ $request_body = array(
	2446	+ 'model' => 'models/' . $selected_model,
	2447	+ 'content' => array(
	2448	+ 'parts' => array(
	2449	+ array('text' => $text)
	2450	+ )
	2451	+ )
	2452	+ );
4902	2453
4903		- // Prepare request body based on provider
4904		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4905		- // Gemini API format
4906		- $request_body = array(
4907		- 'model' => 'models/' . $selected_model,
4908		- 'content' => array(
4909		- 'parts' => array(
4910		- array('text' => $text)
4911		- )
4912		- )
4913		- );
	2454	+ // Set output dimensionality to 1536 for consistency with other models
	2455	+ $request_body['outputDimensionality'] = 1536;
	2456	+ } else {
	2457	+ // OpenAI/Voyage API format
	2458	+ $request_body = array(
	2459	+ 'model' => $selected_model,
	2460	+ 'input' => $text
	2461	+ );
4914	2462
4915		- // Set output dimensionality to 1536 for consistency with other models
4916		- $request_body['outputDimensionality'] = 1536;
4917		- } else {
4918		- // OpenAI/Voyage API format
4919		- $request_body = array(
4920		- 'model' => $selected_model,
4921		- 'input' => $text
4922		- );
	2463	+ // Add output_dimension for voyage-3-large model
	2464	+ if ($selected_model === 'voyage-3-large') {
	2465	+ $request_body['output_dimension'] = 2048;
	2466	+ }
	2467	+ }
4923	2468
4924		- // Add output_dimension for voyage-3-large model
4925		- if ($selected_model === 'voyage-3-large') {
4926		- $request_body['output_dimension'] = 2048;
4927		- }
4928		- }
	2469	+ //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
4929	2470
4930		- //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
	2471	+ // Prepare headers based on provider
	2472	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2473	+ // Gemini uses API key as query parameter
	2474	+ $endpoint .= '?key=' . $api_key;
	2475	+ $headers = array(
	2476	+ 'Content-Type' => 'application/json'
	2477	+ );
	2478	+ } else {
	2479	+ // OpenAI/Voyage use Bearer token
	2480	+ $headers = array(
	2481	+ 'Authorization' => 'Bearer ' . $api_key,
	2482	+ 'Content-Type' => 'application/json'
	2483	+ );
	2484	+ }
4931	2485
4932		- // Prepare headers based on provider
4933		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4934		- // Gemini uses API key as query parameter
4935		- $endpoint .= '?key=' . $api_key;
4936		- $headers = array(
4937		- 'Content-Type' => 'application/json'
4938		- );
4939		- } else {
4940		- // OpenAI/Voyage use Bearer token
4941		- $headers = array(
4942		- 'Authorization' => 'Bearer ' . $api_key,
4943		- 'Content-Type' => 'application/json'
4944		- );
4945		- }
	2486	+ // Make API request
	2487	+ //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
	2488	+ $response = wp_remote_post($endpoint, array(
	2489	+ 'body' => wp_json_encode($request_body),
	2490	+ 'headers' => $headers,
	2491	+ 'timeout' => 60 // Increased timeout for large inputs
	2492	+ ));
4946	2493
4947		- // Make API request
4948		- //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
4949		- $response = wp_remote_post($endpoint, array(
4950		- 'body' => wp_json_encode($request_body),
4951		- 'headers' => $headers,
4952		- 'timeout' => 60 // Increased timeout for large inputs
4953		- ));
	2494	+ // Handle wp_remote_post errors
	2495	+ if (is_wp_error($response)) {
	2496	+ $error_message = $response->get_error_message();
	2497	+ //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
	2498	+ return 'Connection error: ' . $error_message;
	2499	+ }
4954	2500
4955		- // Handle wp_remote_post errors
4956		- if (is_wp_error($response)) {
4957		- $error_message = $response->get_error_message();
4958		- //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
4959		- return 'Connection error: ' . $error_message;
4960		- }
	2501	+ // Get and check HTTP response code
	2502	+ $http_code = wp_remote_retrieve_response_code($response);
	2503	+ //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
4961	2504
4962		- // Get and check HTTP response code
4963		- $http_code = wp_remote_retrieve_response_code($response);
4964		- //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
	2505	+ if ($http_code !== 200) {
	2506	+ $error_body = wp_remote_retrieve_body($response);
	2507	+ //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
4965	2508
4966		- if ($http_code !== 200) {
4967		- $error_body = wp_remote_retrieve_body($response);
4968		- //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
	2509	+ // Try to parse error for more details
	2510	+ $error_json = json_decode($error_body, true);
	2511	+ if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
	2512	+ $error_type = $error_json['error']['type'] ?? 'unknown';
	2513	+ $error_message = $error_json['error']['message'] ?? 'No message';
	2514	+ //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
	2515	+ //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
4969	2516
4970		- // Try to parse error for more details
4971		- $error_json = json_decode($error_body, true);
4972		- if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
4973		- $error_type = $error_json['error']['type'] ?? 'unknown';
4974		- $error_message = $error_json['error']['message'] ?? 'No message';
4975		- //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
4976		- //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
	2517	+ // Customize error message for common API errors
	2518	+ if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
	2519	+ $error_message = sprintf('Invalid %s API key. Please check your API key in the MxChat settings.', $provider_name);
	2520	+ } elseif ($error_type === 'authentication_error') {
	2521	+ $error_message = sprintf('%s authentication failed. Please verify your API key in the MxChat settings.', $provider_name);
	2522	+ }
4977	2523
4978		- // Keep the provider's own diagnostic — a restricted-key 401 names the
4979		- // exact missing scope, and replacing it with "check your API key" sent
4980		- // a customer to regenerate two keys (plan 46b596). Same shape as
4981		- // MxChat_Utils::embedding_failure_error() so both ingestion paths read
4982		- // identically. Key never appears in provider messages, but scrub anyway.
4983		- if ($error_type === 'invalid_request_error' || $error_type === 'authentication_error') {
4984		- if (is_string($api_key) && $api_key !== '') {
4985		- $error_message = str_replace($api_key, '[redacted]', $error_message);
4986		- }
4987		- $error_message = sprintf(
4988		- 'Embedding failed (%s, HTTP %d): %s',
4989		- $selected_model,
4990		- $http_code,
4991		- substr($error_message, 0, 300)
4992		- );
4993		- }
	2524	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2525	+ return $error_message;
	2526	+ }
4994	2527
4995		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4996		- return $error_message;
4997		- }
	2528	+ $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding", $http_code);
	2529	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2530	+ return $error_message;
	2531	+ }
4998	2532
4999		- $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding for bot %s", $http_code, $bot_id);
5000		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
5001		- return $error_message;
5002		- }
	2533	+ // Parse response body
	2534	+ $response_body = wp_remote_retrieve_body($response);
	2535	+ //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
5003	2536
5004		- // Parse response body
5005		- $response_body = wp_remote_retrieve_body($response);
5006		- //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
	2537	+ $response_data = json_decode($response_body, true);
5007	2538
5008		- $response_data = json_decode($response_body, true);
	2539	+ if (json_last_error() !== JSON_ERROR_NONE) {
	2540	+ $error = json_last_error_msg();
	2541	+ //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
	2542	+ //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
	2543	+ return "Failed to parse API response: $error";
	2544	+ }
5009	2545
5010		- if (json_last_error() !== JSON_ERROR_NONE) {
5011		- $error = json_last_error_msg();
5012		- //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
5013		- //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
5014		- return "Failed to parse API response: $error";
5015		- }
	2546	+ // Handle different response formats based on provider
	2547	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2548	+ // Gemini API response format
	2549	+ if (isset($response_data['embedding']['values'])) {
	2550	+ $embedding_dimensions = count($response_data['embedding']['values']);
	2551	+ //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
5016	2552
5017		- // Handle different response formats based on provider
5018		- if (strpos($selected_model, 'gemini-embedding') === 0) {
5019		- // Gemini API response format
5020		- if (isset($response_data['embedding']['values'])) {
5021		- $embedding_dimensions = count($response_data['embedding']['values']);
5022		- //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
	2553	+ // Check if embedding dimensions are as expected (should be 1536)
	2554	+ if ($embedding_dimensions !== 1536) {
	2555	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
	2556	+ }
5023	2557
5024		- // Check if embedding dimensions are as expected (should be 1536)
5025		- if ($embedding_dimensions !== 1536) {
5026		- //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
5027		- }
	2558	+ return $response_data['embedding']['values'];
	2559	+ } else {
	2560	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
	2561	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
5028	2562
5029		- return $response_data['embedding']['values'];
5030		- } else {
5031		- //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
5032		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2563	+ if (isset($response_data['error'])) {
	2564	+ $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
	2565	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2566	+ return $error_message;
	2567	+ }
5033	2568
5034		- if (isset($response_data['error'])) {
5035		- $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
5036		- //error_log('[MXCHAT-EMBED] ' . $error_message);
5037		- return $error_message;
5038		- }
	2569	+ $error_message = "Invalid Gemini API response format: No embedding found";
	2570	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2571	+ return $error_message;
	2572	+ }
	2573	+ } else {
	2574	+ // OpenAI/Voyage API response format
	2575	+ if (isset($response_data['data'][0]['embedding'])) {
	2576	+ $embedding_dimensions = count($response_data['data'][0]['embedding']);
	2577	+ //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
5039	2578
5040		- $error_message = "Invalid Gemini API response format: No embedding found";
5041		- //error_log('[MXCHAT-EMBED] ' . $error_message);
5042		- return $error_message;
5043		- }
5044		- } else {
5045		- // OpenAI/Voyage API response format
5046		- if (isset($response_data['data'][0]['embedding'])) {
5047		- $embedding_dimensions = count($response_data['data'][0]['embedding']);
5048		- //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
	2579	+ // Check if embedding dimensions are as expected
	2580	+ if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
	2581	+ ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
	2582	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
	2583	+ }
5049	2584
5050		- // Check if embedding dimensions are as expected
5051		- if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
5052		- ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
5053		- //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
5054		- }
	2585	+ return $response_data['data'][0]['embedding'];
	2586	+ } else {
	2587	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
	2588	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
5055	2589
5056		- return $response_data['data'][0]['embedding'];
5057		- } else {
5058		- //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
5059		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2590	+ if (isset($response_data['error'])) {
	2591	+ $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
	2592	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2593	+ return $error_message;
	2594	+ }
5060	2595
5061		- if (isset($response_data['error'])) {
5062		- $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
5063		- //error_log('[MXCHAT-EMBED] ' . $error_message);
5064		- return $error_message;
5065		- }
5066		-
5067		- $error_message = "Invalid API response format: No embedding found";
5068		- //error_log('[MXCHAT-EMBED] ' . $error_message);
5069		- return $error_message;
5070		- }
5071		- }
5072		-}
5073		-
5074		-/**
5075		- * Get bot-specific options for multi-bot functionality
5076		- * Falls back to default options if bot_id is 'default' or multi-bot add-on is not active
5077		- */
5078		-private function get_bot_options($bot_id = 'default') {
5079		- //error_log("MXCHAT DEBUG: get_bot_options called for bot: " . $bot_id);
5080		-
5081		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
5082		- //error_log("MXCHAT DEBUG: Using default options (no multi-bot or bot is 'default')");
5083		- return array();
5084		- }
5085		-
5086		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
5087		-
5088		- if (!empty($bot_options)) {
5089		- //error_log("MXCHAT DEBUG: Got bot-specific options from filter");
5090		- if (isset($bot_options['similarity_threshold'])) {
5091		- //error_log(" - similarity_threshold: " . $bot_options['similarity_threshold']);
5092		- }
5093		- }
5094		-
5095		- return is_array($bot_options) ? $bot_options : array();
5096		-}
5097		-
5098		-/**
5099		- * Get bot-specific Pinecone configuration
5100		- * Used in the knowledge retrieval functions
5101		- */
5102		-// Also add debugging to your get_bot_pinecone_config function
5103		-private function get_bot_pinecone_config($bot_id = 'default') {
5104		- //error_log("MXCHAT DEBUG: get_bot_pinecone_config called for bot: " . $bot_id);
5105		-
5106		- // If default bot or multi-bot add-on not active, use default Pinecone config
5107		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
5108		- //error_log("MXCHAT DEBUG: Using default Pinecone config (no multi-bot or bot is 'default')");
5109		- $addon_options = get_option('mxchat_pinecone_addon_options', array());
5110		- $config = array(
5111		- 'use_pinecone' => (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1'),
5112		- 'api_key' => $addon_options['mxchat_pinecone_api_key'] ?? '',
5113		- 'host' => $addon_options['mxchat_pinecone_host'] ?? '',
5114		- 'namespace' => $addon_options['mxchat_pinecone_namespace'] ?? ''
5115		- );
5116		- //error_log("MXCHAT DEBUG: Default config - use_pinecone: " . ($config['use_pinecone'] ? 'true' : 'false'));
5117		- return $config;
5118		- }
5119		-
5120		- //error_log("MXCHAT DEBUG: Calling filter 'mxchat_get_bot_pinecone_config' for bot: " . $bot_id);
5121		-
5122		- // Hook for multi-bot add-on to provide bot-specific Pinecone config
5123		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
5124		-
5125		- if (!empty($bot_pinecone_config)) {
5126		- //error_log("MXCHAT DEBUG: Got bot-specific config from filter");
5127		- //error_log(" - use_pinecone: " . (isset($bot_pinecone_config['use_pinecone']) ? ($bot_pinecone_config['use_pinecone'] ? 'true' : 'false') : 'not set'));
5128		- //error_log(" - host: " . ($bot_pinecone_config['host'] ?? 'not set'));
5129		- //error_log(" - namespace: " . ($bot_pinecone_config['namespace'] ?? 'not set'));
5130		- } else {
5131		- //error_log("MXCHAT DEBUG: Filter returned empty config!");
5132		- }
5133		-
5134		- return is_array($bot_pinecone_config) ? $bot_pinecone_config : array();
5135		-}
5136		-
5137		-
	2596	+ $error_message = "Invalid API response format: No embedding found";
	2597	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2598	+ return $error_message;
	2599	+ }
	2600	+ }
	2601	+ }
5138	2602	public function mxchat_ajax_dismiss_completed_status() {
5139	2603	try {
5140	2604	// Verify the request
5141	2605	check_ajax_referer('mxchat_status_nonce', 'nonce');
		@@ -5254,10 +2718,10 @@
5254	2718
5255	2719	$html .= '<p><strong>' . esc_html__('Status:', 'mxchat') . '</strong> ' . esc_html(ucfirst($status['status'])) . '</p>';
5256	2720	$html .= '<p><strong>' . esc_html__('Last update:', 'mxchat') . '</strong> ' . esc_html($status['last_update']) . '</p>';
5257	2721
5258		- // Add completion summary if available AND it's an array
5259		- if (isset($status['completion_summary']) && is_array($status['completion_summary']) && !empty($status['completion_summary'])) {
	2722	+ // Add completion summary if available
	2723	+ if (isset($status['completion_summary'])) {
5260	2724	$summary = $status['completion_summary'];
5261	2725	$html .= '<div class="mxchat-completion-summary">';
5262	2726	$html .= '<h5>' . esc_html__('Processing Summary', 'mxchat') . '</h5>';
5263	2727	$html .= '<p><strong>' . esc_html__('Total Pages:', 'mxchat') . '</strong> ' . esc_html($summary['total_pages']) . '</p>';
		@@ -5266,10 +2730,10 @@
5266	2730	$html .= '<p><strong>' . esc_html__('Completed:', 'mxchat') . '</strong> ' . esc_html($summary['completion_time']) . '</p>';
5267	2731	$html .= '</div>';
5268	2732	}
5269	2733
5270		- // Add failed pages list if any AND it's an array
5271		- if (isset($status['failed_pages_list']) && is_array($status['failed_pages_list']) && !empty($status['failed_pages_list'])) {
	2734	+ // Add failed pages list if any
	2735	+ if (!empty($status['failed_pages_list'])) {
5272	2736	$html .= $this->mxchat_render_failed_pages_list($status['failed_pages_list']);
5273	2737	}
5274	2738
5275	2739	// Add error message if any
		@@ -5283,8 +2747,9 @@
5283	2747	$html .= '</div>'; // End card
5284	2748
5285	2749	return $html;
5286	2750	}
	2751	+
5287	2752	/**
5288	2753	* Render sitemap status card HTML
5289	2754	*/
5290	2755	private function mxchat_render_sitemap_status_card($status, $sitemap_url) {
		@@ -5340,10 +2805,10 @@
5340	2805
5341	2806	$html .= '<p><strong>' . esc_html__('Status:', 'mxchat') . '</strong> ' . esc_html(ucfirst($status['status'])) . '</p>';
5342	2807	$html .= '<p><strong>' . esc_html__('Last update:', 'mxchat') . '</strong> ' . esc_html($status['last_update']) . '</p>';
5343	2808
5344		- // Add completion summary if available AND it's an array
5345		- if (isset($status['completion_summary']) && is_array($status['completion_summary']) && !empty($status['completion_summary'])) {
	2809	+ // Add completion summary if available
	2810	+ if (isset($status['completion_summary'])) {
5346	2811	$summary = $status['completion_summary'];
5347	2812	$html .= '<div class="mxchat-completion-summary">';
5348	2813	$html .= '<h5>' . esc_html__('Processing Summary', 'mxchat') . '</h5>';
5349	2814	$html .= '<p><strong>' . esc_html__('Total URLs:', 'mxchat') . '</strong> ' . esc_html($summary['total_urls']) . '</p>';
		@@ -5372,16 +2837,13 @@
5372	2837	$html .= '</div>'; // End card
5373	2838
5374	2839	return $html;
5375	2840	}
5376		-
5377		-
5378	2841	/**
5379	2842	* Render failed pages list
5380	2843	*/
5381	2844	private function mxchat_render_failed_pages_list($failed_pages_list) {
5382		- // Validate that $failed_pages_list is an array and not empty
5383		- if (!is_array($failed_pages_list) || empty($failed_pages_list)) {
	2845	+ if (empty($failed_pages_list)) {
5384	2846	return '';
5385	2847	}
5386	2848
5387	2849	$html = '<div class="mxchat-error-notice">';
		@@ -5406,13 +2868,8 @@
5406	2868	return ($b['time'] ?? 0) - ($a['time'] ?? 0);
5407	2869	});
5408	2870
5409	2871	foreach ($sorted_failed_pages as $item) {
5410		- // Ensure $item is an array before accessing its elements
5411		- if (!is_array($item)) {
5412		- continue;
5413		- }
5414		-
5415	2872	$time_ago = isset($item['time']) ? human_time_diff($item['time'], current_time('timestamp')) . ' ' . esc_html__('ago', 'mxchat') : 'Unknown';
5416	2873	$html .= '<tr>';
5417	2874	$html .= '<td>' . esc_html__('Page', 'mxchat') . ' ' . esc_html($item['page'] ?? 'Unknown') . '</td>';
5418	2875	$html .= '<td style="word-break: break-word;">' . esc_html($item['error'] ?? 'Unknown error') . '</td>';
		@@ -5430,10 +2887,9 @@
5430	2887	/**
5431	2888	* Render failed URLs list
5432	2889	*/
5433	2890	private function mxchat_render_failed_urls_list($failed_urls_list) {
5434		- // Validate that $failed_urls_list is an array and not empty
5435		- if (!is_array($failed_urls_list) || empty($failed_urls_list)) {
	2891	+ if (empty($failed_urls_list)) {
5436	2892	return '';
5437	2893	}
5438	2894
5439	2895	$html = '<div class="mxchat-failed-urls-container">';
		@@ -5460,13 +2916,8 @@
5460	2916	// Show up to 50 failed URLs
5461	2917	$display_urls = array_slice($sorted_failed_urls, 0, 50);
5462	2918
5463	2919	foreach ($display_urls as $item) {
5464		- // Ensure $item is an array before accessing its elements
5465		- if (!is_array($item)) {
5466		- continue;
5467		- }
5468		-
5469	2920	$url = $item['url'] ?? '';
5470	2921	$time_ago = isset($item['time']) ? human_time_diff($item['time'], current_time('timestamp')) . ' ' . esc_html__('ago', 'mxchat') : 'Unknown';
5471	2922
5472	2923	// Truncate URL for display
		@@ -5498,572 +2949,18 @@
5498	2949
5499	2950	return $html;
5500	2951	}
5501	2952
5502		-/**
5503		- * Get all ACF fields for a specific post, excluding any fields the user has disabled
5504		- */
5505		-public function mxchat_get_acf_fields_for_post($post_id) {
5506		- if (!function_exists('get_fields')) {
5507		- return array();
5508		- }
5509		-
5510		- $fields = get_fields($post_id);
5511		- if (!$fields || !is_array($fields)) {
5512		- return array();
5513		- }
5514		-
5515		- // Get excluded fields from settings
5516		- $excluded_fields = get_option('mxchat_acf_excluded_fields', array());
5517		- if (!empty($excluded_fields) && is_array($excluded_fields)) {
5518		- foreach ($excluded_fields as $excluded_field) {
5519		- if (isset($fields[$excluded_field])) {
5520		- unset($fields[$excluded_field]);
5521		- }
5522		- }
5523		- }
5524		-
5525		- return $fields;
5526		-}
5527		-
5528		-/**
5529		- * Get all registered ACF field groups and their fields for the settings UI
5530		- */
5531		-public function mxchat_get_all_acf_fields() {
5532		- if (!function_exists('acf_get_field_groups') || !function_exists('acf_get_fields')) {
5533		- return array();
5534		- }
5535		-
5536		- $all_fields = array();
5537		- $field_groups = acf_get_field_groups();
5538		-
5539		- if (!empty($field_groups)) {
5540		- foreach ($field_groups as $group) {
5541		- $group_fields = acf_get_fields($group['key']);
5542		- if (!empty($group_fields)) {
5543		- $all_fields[$group['title']] = array();
5544		- foreach ($group_fields as $field) {
5545		- $all_fields[$group['title']][] = array(
5546		- 'name' => $field['name'],
5547		- 'label' => $field['label'],
5548		- 'type' => $field['type']
5549		- );
5550		- }
5551		- }
5552		- }
5553		- }
5554		-
5555		- return $all_fields;
5556		-}
5557		-
5558		-/**
5559		- * Get whitelisted custom post meta for a given post
5560		- * This allows non-ACF meta fields (like OptionTree, theme meta boxes) to be included in embeddings
5561		- */
5562		-public function mxchat_get_whitelisted_post_meta($post_id) {
5563		- $whitelist = get_option('mxchat_custom_meta_whitelist', '');
5564		-
5565		- if (empty($whitelist)) {
5566		- return array();
5567		- }
5568		-
5569		- // Parse the whitelist - one meta key per line
5570		- $meta_keys = array_filter(array_map('trim', explode("\n", $whitelist)));
5571		-
5572		- if (empty($meta_keys)) {
5573		- return array();
5574		- }
5575		-
5576		- $result = array();
5577		-
5578		- foreach ($meta_keys as $key) {
5579		- // Skip empty keys
5580		- if (empty($key)) {
5581		- continue;
5582		- }
5583		-
5584		- $value = get_post_meta($post_id, $key, true);
5585		-
5586		- // Only include non-empty string values
5587		- if (!empty($value) && is_string($value)) {
5588		- $result[$key] = $value;
5589		- } elseif (!empty($value) && is_array($value)) {
5590		- // Handle array values by joining them
5591		- $flat_value = $this->mxchat_flatten_meta_array($value);
5592		- if (!empty($flat_value)) {
5593		- $result[$key] = $flat_value;
5594		- }
5595		- }
5596		- }
5597		-
5598		- return $result;
5599		-}
5600		-
5601		-/**
5602		- * Flatten array meta values into a readable string
5603		- */
5604		-private function mxchat_flatten_meta_array($array, $depth = 0) {
5605		- if ($depth > 3) {
5606		- return ''; // Prevent infinite recursion
5607		- }
5608		-
5609		- $parts = array();
5610		-
5611		- foreach ($array as $key => $value) {
5612		- if (is_string($value) && !empty($value)) {
5613		- $parts[] = $value;
5614		- } elseif (is_array($value)) {
5615		- $nested = $this->mxchat_flatten_meta_array($value, $depth + 1);
5616		- if (!empty($nested)) {
5617		- $parts[] = $nested;
5618		- }
5619		- }
5620		- }
5621		-
5622		- return implode(', ', $parts);
5623		-}
5624		-
5625		-/**
5626		- * Format ACF field values for content extraction
5627		- */
5628		-public function mxchat_format_acf_field_value($value, $field_name = '', $post_id = 0) {
5629		- if (empty($value)) {
5630		- return '';
5631		- }
5632		-
5633		- // Handle WP_Post objects first (THIS IS THE KEY FIX)
5634		- if ($value instanceof WP_Post) {
5635		- return $value->post_title ?: '';
5636		- }
5637		-
5638		- // Handle other WP objects
5639		- if (is_object($value)) {
5640		- if (isset($value->post_title)) {
5641		- return $value->post_title;
5642		- } elseif (isset($value->display_name)) {
5643		- return $value->display_name;
5644		- } elseif (isset($value->name)) {
5645		- return $value->name;
5646		- } elseif (method_exists($value, '__toString')) {
5647		- try {
5648		- return (string) $value;
5649		- } catch (Exception $e) {
5650		- return '';
5651		- }
5652		- }
5653		- // For any other objects, return empty string
5654		- return '';
5655		- }
5656		-
5657		- // Handle different ACF field types
5658		- if (is_array($value)) {
5659		- // Check if it's an image/file field
5660		- if (isset($value['url'])) {
5661		- // Image field - return alt text, title, or caption
5662		- if (!empty($value['alt'])) {
5663		- return $value['alt'];
5664		- } elseif (!empty($value['title'])) {
5665		- return $value['title'];
5666		- } elseif (!empty($value['caption'])) {
5667		- return $value['caption'];
5668		- } else {
5669		- return ''; // Don't include just the URL
5670		- }
5671		- }
5672		-
5673		- // Check if it's a post object or relationship field
5674		- if (isset($value['post_title'])) {
5675		- return $value['post_title'];
5676		- }
5677		-
5678		- // Check if it's a user field
5679		- if (isset($value['display_name'])) {
5680		- return $value['display_name'];
5681		- }
5682		-
5683		- // Check if it's a taxonomy term
5684		- if (isset($value['name']) && isset($value['taxonomy'])) {
5685		- return $value['name'];
5686		- }
5687		-
5688		- // Check if it's a select field with label
5689		- if (isset($value['label'])) {
5690		- return $value['label'];
5691		- }
5692		-
5693		- // Check for repeater field or flexible content
5694		- if (is_numeric(key($value))) {
5695		- $sub_values = array();
5696		- foreach ($value as $sub_item) {
5697		- if (is_array($sub_item)) {
5698		- // For repeater/flexible content, extract text values
5699		- $sub_text = $this->mxchat_extract_text_from_acf_array($sub_item);
5700		- if (!empty($sub_text)) {
5701		- $sub_values[] = $sub_text;
5702		- }
5703		- } elseif ($sub_item instanceof WP_Post) {
5704		- // Handle WP_Post objects in arrays
5705		- $sub_values[] = $sub_item->post_title ?: '';
5706		- } else {
5707		- $sub_values[] = (string) $sub_item;
5708		- }
5709		- }
5710		- return implode(', ', array_filter($sub_values));
5711		- }
5712		-
5713		- // For other arrays, try to extract meaningful text
5714		- $text_values = array();
5715		- foreach ($value as $key => $val) {
5716		- if (is_string($val) && !empty(trim($val))) {
5717		- $text_values[] = trim($val);
5718		- } elseif ($val instanceof WP_Post) {
5719		- // Handle WP_Post objects in associative arrays
5720		- $text_values[] = $val->post_title ?: '';
5721		- } elseif (is_array($val) && isset($val['post_title'])) {
5722		- $text_values[] = $val['post_title'];
5723		- } elseif (is_array($val) && isset($val['name'])) {
5724		- $text_values[] = $val['name'];
5725		- }
5726		- }
5727		-
5728		- return implode(', ', array_filter($text_values));
5729		- }
5730		-
5731		- // Handle boolean values
5732		- if (is_bool($value)) {
5733		- return $value ? 'Yes' : 'No';
5734		- }
5735		-
5736		- // Handle numeric values
5737		- if (is_numeric($value)) {
5738		- return (string) $value;
5739		- }
5740		-
5741		- // Handle string values
5742		- if (is_string($value)) {
5743		- return trim($value);
5744		- }
5745		-
5746		- // For anything else that we can't handle, return empty string
5747		- // This prevents the "Object could not be converted to string" error
5748		- return '';
5749		-}
5750		-
5751		-/**
5752		- * Extract text from complex ACF array structures
5753		- */
5754		-private function mxchat_extract_text_from_acf_array($array) {
5755		- if (!is_array($array)) {
5756		- return '';
5757		- }
5758		-
5759		- $text_parts = array();
5760		-
5761		- foreach ($array as $key => $value) {
5762		- if (is_string($value) && !empty(trim($value))) {
5763		- // Skip keys that are likely to be IDs or technical values
5764		- if (!is_numeric($value) || strlen($value) > 10) {
5765		- $text_parts[] = trim($value);
5766		- }
5767		- } elseif ($value instanceof WP_Post) {
5768		- // Handle WP_Post objects
5769		- $text_parts[] = $value->post_title ?: '';
5770		- } elseif (is_array($value)) {
5771		- if (isset($value['post_title'])) {
5772		- $text_parts[] = $value['post_title'];
5773		- } elseif (isset($value['name'])) {
5774		- $text_parts[] = $value['name'];
5775		- } elseif (isset($value['label'])) {
5776		- $text_parts[] = $value['label'];
5777		- }
5778		- } elseif (is_object($value)) {
5779		- // Handle other objects safely
5780		- if (isset($value->post_title)) {
5781		- $text_parts[] = $value->post_title;
5782		- } elseif (isset($value->name)) {
5783		- $text_parts[] = $value->name;
5784		- } elseif (isset($value->display_name)) {
5785		- $text_parts[] = $value->display_name;
5786		- }
5787		- }
5788		- }
5789		-
5790		- return implode(', ', array_filter($text_parts));
5791		-}
5792		-
5793		-/**
5794		- * Walk an ACF field value tree and collect attachment IDs for any value that
5795		- * resolves to a PDF in the WordPress media library. Handles the three shapes
5796		- * ACF returns for File/Image/URL fields (array with ID+url, integer attachment ID,
5797		- * plain URL string), and recurses through repeater/group/flexible content.
5798		- *
5799		- * @param mixed $value The ACF field value (any depth)
5800		- * @param array $out Accumulator (passed by reference) for attachment IDs
5801		- * @param int $depth Recursion guard
5802		- */
5803		-private function mxchat_collect_pdf_attachment_ids_from_acf_value($value, &$out, $depth = 0) {
5804		- if ($depth > 6) {
5805		- return; // prevent runaway recursion on circular/very-deep structures
5806		- }
5807		-
5808		- if (empty($value)) {
5809		- return;
5810		- }
5811		-
5812		- // Array shapes: ACF File/Image return value=array; repeaters/groups are arrays of arrays
5813		- if (is_array($value)) {
5814		- // Direct File/Image-style array (has 'url' and usually 'ID' + 'mime_type')
5815		- $looks_like_attachment = isset($value['url']) || isset($value['ID']) || isset($value['id']);
5816		- if ($looks_like_attachment) {
5817		- $att_id = 0;
5818		- if (!empty($value['ID']) && is_numeric($value['ID'])) {
5819		- $att_id = (int) $value['ID'];
5820		- } elseif (!empty($value['id']) && is_numeric($value['id'])) {
5821		- $att_id = (int) $value['id'];
5822		- } elseif (!empty($value['url']) && is_string($value['url'])) {
5823		- $att_id = (int) attachment_url_to_postid($value['url']);
5824		- }
5825		-
5826		- $is_pdf = false;
5827		- if (!empty($value['mime_type']) && $value['mime_type'] === 'application/pdf') {
5828		- $is_pdf = true;
5829		- } elseif (!empty($value['subtype']) && strtolower((string) $value['subtype']) === 'pdf') {
5830		- $is_pdf = true;
5831		- } elseif (!empty($value['url']) && is_string($value['url']) && $this->mxchat_url_looks_like_pdf($value['url'])) {
5832		- $is_pdf = true;
5833		- } elseif ($att_id && get_post_mime_type($att_id) === 'application/pdf') {
5834		- $is_pdf = true;
5835		- }
5836		-
5837		- if ($is_pdf && $att_id && get_post_mime_type($att_id) === 'application/pdf') {
5838		- $out[] = $att_id;
5839		- }
5840		- // An array node that represents one attachment doesn't contain other
5841		- // attachments inside it — done with this branch.
5842		- return;
5843		- }
5844		-
5845		- // Recurse: repeater rows, flexible-content layouts, groups, etc.
5846		- foreach ($value as $sub) {
5847		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($sub, $out, $depth + 1);
5848		- }
5849		- return;
5850		- }
5851		-
5852		- // Plain numeric attachment ID (ACF File field set to "Return: ID")
5853		- if (is_numeric($value)) {
5854		- $att_id = (int) $value;
5855		- if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5856		- $out[] = $att_id;
5857		- }
5858		- return;
5859		- }
5860		-
5861		- // Plain string — URL pointing at a PDF (ACF File field set to "Return: URL", or a custom URL/text field)
5862		- if (is_string($value)) {
5863		- $trimmed = trim($value);
5864		- if ($trimmed !== '' && $this->mxchat_url_looks_like_pdf($trimmed)) {
5865		- $att_id = (int) attachment_url_to_postid($trimmed);
5866		- if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5867		- $out[] = $att_id;
5868		- }
5869		- }
5870		- return;
5871		- }
5872		-}
5873		-
5874		-/**
5875		- * Heuristic: does this URL/string look like a PDF reference?
5876		- * Tolerates query strings and fragments (#page=2).
5877		- */
5878		-private function mxchat_url_looks_like_pdf($url) {
5879		- if (!is_string($url) || $url === '') {
5880		- return false;
5881		- }
5882		- // Strip query + fragment before checking extension
5883		- $path = preg_replace('/[?#].*$/', '', $url);
5884		- return (bool) preg_match('/\.pdf$/i', $path);
5885		-}
5886		-
5887		-/**
5888		- * Extract text from a PDF attachment by ID using the bundled Smalot parser.
5889		- * Reads the file directly from disk via get_attached_file (no HTTP fetch).
5890		- * Result is cached on the attachment as post_meta keyed by file mtime so we
5891		- * only parse the same PDF once unless the file changes on disk.
5892		- *
5893		- * @param int $attachment_id
5894		- * @return string Extracted plain text, or '' on failure.
5895		- */
5896		-private function mxchat_extract_pdf_text_by_attachment_id($attachment_id) {
5897		- $attachment_id = (int) $attachment_id;
5898		- if ($attachment_id <= 0) {
5899		- return '';
5900		- }
5901		- if (get_post_mime_type($attachment_id) !== 'application/pdf') {
5902		- return '';
5903		- }
5904		-
5905		- $pdf_path = get_attached_file($attachment_id);
5906		- if (empty($pdf_path) || !file_exists($pdf_path) || !is_readable($pdf_path)) {
5907		- return '';
5908		- }
5909		-
5910		- // Raw-file size cap. Parsing very large PDFs can OOM the request; skip with a log entry
5911		- // and let the rest of the ACF content land in the KB. Filterable for users who need it bigger.
5912		- $default_max_bytes = 25 * 1024 * 1024;
5913		- $max_bytes = (int) apply_filters('mxchat_acf_pdf_max_bytes', $default_max_bytes, $attachment_id, $pdf_path);
5914		- if ($max_bytes > 0) {
5915		- $file_size = @filesize($pdf_path);
5916		- if ($file_size !== false && $file_size > $max_bytes) {
5917		- error_log(sprintf(
5918		- '[mxchat] ACF PDF skipped (over size cap): attachment %d "%s" %d bytes > cap %d',
5919		- $attachment_id,
5920		- basename($pdf_path),
5921		- $file_size,
5922		- $max_bytes
5923		- ));
5924		- return '';
5925		- }
5926		- }
5927		-
5928		- $mtime = @filemtime($pdf_path);
5929		- $cache_meta_key = '_mxchat_acf_pdf_text_v1';
5930		- $cached = get_post_meta($attachment_id, $cache_meta_key, true);
5931		- if (is_array($cached) && isset($cached['mtime'], $cached['text']) && (int) $cached['mtime'] === (int) $mtime) {
5932		- return (string) $cached['text'];
5933		- }
5934		-
5935		- $text = '';
5936		- try {
5937		- if (function_exists('mxchat_load_pdf_parser')) {
5938		- mxchat_load_pdf_parser();
5939		- }
5940		- if (!class_exists('\\Smalot\\PdfParser\\Parser')) {
5941		- return '';
5942		- }
5943		- $parser = new \Smalot\PdfParser\Parser();
5944		- $pdf = $parser->parseFile($pdf_path);
5945		- $pages = $pdf->getPages();
5946		- $page_texts = array();
5947		- $acf_page_num = 0;
5948		- foreach ($pages as $page) {
5949		- $acf_page_num++;
5950		- $page_text = '';
5951		- try {
5952		- $page_text = $page->getText();
5953		- } catch (\Exception $e) {
5954		- $page_text = '';
5955		- }
5956		- if (!empty($page_text)) {
5957		- $page_text = MxChat_Utils::normalize_pdf_rtl($page_text, 'acf_pdf attachment ' . $attachment_id . ' page ' . $acf_page_num);
5958		- $page_texts[] = $page_text;
5959		- }
5960		- }
5961		- $text = trim(implode("\n\n", $page_texts));
5962		- } catch (\Exception $e) {
5963		- error_log('[mxchat] ACF PDF extraction failed for attachment ' . $attachment_id . ': ' . $e->getMessage());
5964		- return '';
5965		- } catch (\Throwable $e) {
5966		- error_log('[mxchat] ACF PDF extraction error for attachment ' . $attachment_id . ': ' . $e->getMessage());
5967		- return '';
5968		- }
5969		-
5970		- // Cap per-PDF text to avoid blowing up the embedding payload on enormous PDFs.
5971		- // The chunker downstream will still split this into multiple vectors.
5972		- $max_len = (int) apply_filters('mxchat_acf_pdf_text_max_length', 50000);
5973		- if ($max_len > 0 && strlen($text) > $max_len) {
5974		- $text = substr($text, 0, $max_len);
5975		- }
5976		-
5977		- update_post_meta($attachment_id, $cache_meta_key, array(
5978		- 'mtime' => (int) $mtime,
5979		- 'text' => $text,
5980		- ));
5981		-
5982		- return $text;
5983		-}
5984		-
5985		-/**
5986		- * Handle ACF save - fires after ACF fields are saved
5987		- * This ensures ACF field data is available when syncing to knowledge base
5988		- */
5989		-public function mxchat_handle_acf_save($post_id) {
5990		- // Skip if not a valid post
5991		- if (!$post_id || $post_id === 'options') {
5992		- return;
5993		- }
5994		-
5995		- // Skip autosaves and revisions
	2953	+public function mxchat_handle_post_update($post_id, $post, $update) {
	2954	+ // Basic validation checks
5996	2955	if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5997	2956	return;
5998	2957	}
5999		-
6000		- $post = get_post($post_id);
6001		- if (!$post) {
6002		- return;
6003		- }
6004		-
6005		- $post_type = $post->post_type;
6006		-
6007		- // Check if sync is enabled for this post type
6008		- $should_sync = false;
6009		-
6010		- if ($post_type === 'post' && get_option('mxchat_auto_sync_posts') === '1') {
6011		- $should_sync = true;
6012		- } else if ($post_type === 'page' && get_option('mxchat_auto_sync_pages') === '1') {
6013		- $should_sync = true;
6014		- } else if ($post_type === 'product' && class_exists('WooCommerce')) {
6015		- // WooCommerce products - check if WooCommerce integration is enabled
6016		- $options = get_option('mxchat_options', array());
6017		- if (isset($options['enable_woocommerce_integration']) &&
6018		- ($options['enable_woocommerce_integration'] === '1' || $options['enable_woocommerce_integration'] === 'on')) {
6019		- $should_sync = true;
6020		- }
6021		- } else {
6022		- // Check custom post types
6023		- $option_name = 'mxchat_auto_sync_' . $post_type;
6024		- if (get_option($option_name) === '1') {
6025		- $should_sync = true;
6026		- }
6027		- }
6028		-
6029		- if (!$should_sync) {
6030		- return;
6031		- }
6032		-
6033		- // Only process published posts
	2958	+
	2959	+ // Only process published content
6034	2960	if ($post->post_status !== 'publish') {
6035	2961	return;
6036	2962	}
6037		-
6038		- // Check if this post has any ACF fields - if not, no need to re-sync
6039		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
6040		- if (empty($acf_fields)) {
6041		- return;
6042		- }
6043		-
6044		- // Use a transient to prevent duplicate processing (post_updated may have already run)
6045		- $transient_key = 'mxchat_acf_synced_' . $post_id;
6046		- if (get_transient($transient_key)) {
6047		- return;
6048		- }
6049		- set_transient($transient_key, true, 60); // Prevent re-processing for 60 seconds
6050		-
6051		- // Re-run the sync with ACF data now available
6052		- // We pass $update=true since this is effectively an update with ACF data
6053		- $this->mxchat_handle_post_update($post_id, $post, true);
6054		-}
6055		-
6056		-public function mxchat_handle_post_update($post_id, $post, $update) {
6057		- // The in-flight-update marker has done its job the moment post_updated runs; drop it
6058		- // before any early return so it can never outlive its own save (a failed $wpdb->update
6059		- // inside wp_insert_post returns after pre_post_update but before the transition).
6060		- unset($this->pending_post_update[$post_id]);
6061		-
6062		- // Basic validation checks
6063		- if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
6064		- return;
6065		- }
6066	2963
6067	2964	$post_type = $post->post_type;
6068	2965
6069	2966	// Check if sync is enabled for this post type
		@@ -6085,717 +2982,80 @@
6085	2982	if (!$should_sync) {
6086	2983	return;
6087	2984	}
6088	2985
6089		- // Check if we have stored the previous status and URL in our transients
6090		- $previous_status_key = 'mxchat_prev_status_' . $post_id;
6091		- $previous_status = get_transient($previous_status_key);
	2986	+ // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
	2987	+ $title = get_the_title($post_id);
	2988	+ $content = get_post_field('post_content', $post_id);
6092	2989
6093		- $previous_url_key = 'mxchat_prev_url_' . $post_id;
6094		- $previous_url = get_transient($previous_url_key);
	2990	+ // Apply WordPress content filters to get properly formatted content
	2991	+ $content = apply_filters('the_content', $content);
6095	2992
6096		- // If the post was previously published but is now not published, remove from knowledge base
6097		- if ($previous_status === 'publish' && $post->post_status !== 'publish') {
6098		- // Use the stored URL from when it was published, or fall back to current permalink
6099		- $source_url = $previous_url ?: get_permalink($post_id);
6100		-
6101		- // mxchat_handle_status_transition already deleted for this post earlier in this
6102		- // request (it fires first inside wp_insert_post); skip the redundant round-trip.
6103		- if ($source_url && empty($this->transition_deleted_posts[$post_id])) {
6104		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
6105		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
6106		- }
6107		-
6108		- // Clean up the transients and exit early
6109		- delete_transient($previous_status_key);
6110		- delete_transient($previous_url_key);
6111		- return;
6112		- }
6113		-
6114		- // Slug/permalink rename while still published: delete the old vectors before upserting new ones.
6115		- // Without this, md5(old_url) vectors (base + chunks) would be orphaned under the stale URL.
6116		- if ($post->post_status === 'publish' && !empty($previous_url)) {
6117		- $current_url = get_permalink($post_id);
6118		- if ($current_url && $current_url !== $previous_url) {
6119		- MxChat_Utils::delete_chunks_for_url($previous_url, 'default');
6120		- }
6121		- }
6122		-
6123		- // Store the current status for next time (if this is an update)
6124		- if ($update) {
6125		- set_transient($previous_status_key, $post->post_status, DAY_IN_SECONDS);
6126		-
6127		- // If the post is currently published, also store its URL
6128		- if ($post->post_status === 'publish') {
6129		- $current_url = get_permalink($post_id);
6130		- set_transient($previous_url_key, $current_url, DAY_IN_SECONDS);
6131		- }
6132		- }
	2993	+ // Strip tags but preserve structure
	2994	+ $content = wp_strip_all_tags($content);
6133	2995
6134		- // Only process currently published content for adding/updating.
6135		- // transition_indexed_posts: mxchat_handle_status_transition's arrival edge may have
6136		- // already indexed this post earlier in this request (editor publishes fire
6137		- // transition_post_status first, then post_updated) — skip the duplicate embed.
6138		- // Consume-once: the flag is cleared when honoured, so a LATER save of the same
6139		- // post in one long-running process (WP-CLI scripts, importers) re-indexes normally.
6140		- if ($post->post_status === 'publish') {
6141		- if (!empty($this->transition_indexed_posts[$post_id])) {
6142		- unset($this->transition_indexed_posts[$post_id]);
6143		- } else {
6144		- $this->mxchat_index_published_post($post_id, $post);
	2996	+ // Combine title and content
	2997	+ $final_content = $title . "\n\n" . $content;
	2998	+
	2999	+ // For custom post types like job_listing, include additional fields
	3000	+ if ($post_type === 'job_listing') {
	3001	+ // Add job-specific meta if available
	3002	+ $job_location = get_post_meta($post_id, '_job_location', true);
	3003	+ if (!empty($job_location)) {
	3004	+ $final_content .= "\n\nLocation: " . $job_location;
6145	3005	}
6146		- }
6147		-
6148		- // Clean up the stored previous status if not used above
6149		- if ($previous_status !== 'publish' || $post->post_status === 'publish') {
6150		- delete_transient($previous_status_key);
6151		- delete_transient($previous_url_key);
6152		- }
6153		-}
6154		-
6155		-/**
6156		- * Index a published post into the knowledge base: preprocessing filter, content
6157		- * assembly (title/excerpt/body), WooCommerce product enrichment, job_listing meta,
6158		- * ACF fields (+ optional PDF extraction), whitelisted custom meta, embedding and
6159		- * upsert, then tag-based role restriction.
6160		- *
6161		- * Shared by the post_updated auto-sync path (mxchat_handle_post_update) and the
6162		- * transition_post_status arrival edge (mxchat_handle_status_transition), so
6163		- * scheduled publishes (wp_publish_post) and direct status=publish inserts index
6164		- * identically to editor saves (plan 3055e1). Pure extraction of the former
6165		- * publish branch — body indentation retained to keep the diff reviewable.
6166		- */
6167		-private function mxchat_index_published_post($post_id, $post) {
6168		- $post_type = $post->post_type;
6169		-
6170		- // Get the source URL
6171		- $source_url = get_permalink($post_id);
6172		-
6173		- // A draft published programmatically (wp_publish_post) can reach this
6174		- // point with an EMPTY post_name — wp_insert_post skips slug generation
6175		- // for draft/pending — and get_permalink() then resolves to the bare
6176		- // site root. A knowledge row keyed to the homepage cites the wrong URL
6177		- // and answers homepage questions with this post's body, so refuse to
6178		- // write it; the post indexes correctly on its next save, once the slug
6179		- // exists. The empty-post_name test is what keeps a legitimate static
6180		- // front page (which has a slug but a root permalink) indexable.
6181		- // (Plan d138c4.)
6182		- if ('' === $post->post_name
6183		- && untrailingslashit($source_url) === untrailingslashit(home_url())) {
6184		- return;
6185		- }
6186		-
6187		- /**
6188		- * Allow developers to modify post data before processing into the knowledge base.
6189		- * Same filter and signature as the manual bulk-import path
6190		- * (ajax_mxchat_process_selected_content), so a callback registered once covers
6191		- * every indexing route. Purely additive — zero behaviour change when unhooked.
6192		- * Auto-sync runs under the 'default' bot context, matching the rest of this
6193		- * function.
6194		- *
6195		- * @param WP_Post $post The post about to be indexed.
6196		- * @param string $bot_id Bot context ('default' on auto-sync).
6197		- */
6198		- $post = apply_filters('mxchat_before_process_post', $post, 'default');
6199		- if (!($post instanceof WP_Post)) {
6200		- $post = get_post($post_id); // defend against a bad callback return
6201		- }
6202		-
6203		- // Get content with proper formatting (matching ajax_mxchat_process_selected_content),
6204		- // reading from the FILTERED post object — not re-fetched by ID, which would discard it
6205		- // Raw post_title, NOT get_the_title(): the_title applies wptexturize +
6206		- // convert_chars (curly quotes and em-dashes become HTML entities in the
6207		- // embedded string) and prepends the "Protected:" / "Private:" display
6208		- // chrome. The knowledge base stores facts, not display strings — and the
6209		- // bulk-import path has always read the raw title, so this is also what
6210		- // makes the two paths agree.
6211		- $title = $this->mxchat_decode_entities_for_indexing($post->post_title);
6212		- $content = get_post_field('post_content', $post);
6213		- $excerpt = get_post_field('post_excerpt', $post);
6214		-
6215		- // Remove shortcode tags but preserve content inside them
6216		- $content = $this->strip_shortcode_tags_preserve_content($content);
6217		- $excerpt = $this->strip_shortcode_tags_preserve_content($excerpt);
6218		-
6219		- // Strip tags but preserve structure (don't use 'the_content' filter as it may re-add shortcodes)
6220		- // Entity decode at output time, matching the bulk-import path (d2c92e).
6221		- $content = $this->mxchat_decode_entities_for_indexing(wp_strip_all_tags($content));
6222		-
6223		- // Combine title, short description (if exists), and content
6224		- $final_content = $title . "\n\n";
6225		-
6226		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
6227		- // trim() only in the TEST — see the matching note on the bulk-import path.
6228		- if (trim($excerpt) !== '') {
6229		- $final_content .= "Short Description: " . $this->mxchat_decode_entities_for_indexing(wp_strip_all_tags($excerpt)) . "\n\n";
6230		- }
6231		-
6232		- $final_content .= $content;
6233		-
6234		- // For WooCommerce products, include pricing and product details
6235		- if ($post_type === 'product' && class_exists('WooCommerce')) {
6236		- $product = wc_get_product($post_id);
6237		-
6238		- if ($product) {
6239		- $sku = $product->get_sku();
6240		-
6241		- // Add pricing information (base-currency pinned — see mxchat_product_price_lines)
6242		- $final_content .= "\n";
6243		- $final_content .= $this->mxchat_product_price_lines($product);
6244		-
6245		- if (!empty($sku)) {
6246		- $final_content .= "SKU: " . $sku . "\n";
6247		- }
6248		-
6249		- // Get product categories
6250		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
6251		- if (!empty($categories) && !is_wp_error($categories)) {
6252		- $final_content .= "Categories: " . implode(', ', $categories) . "\n";
6253		- }
	3006	+
	3007	+ // Get job type terms
	3008	+ $job_types = get_the_terms($post_id, 'job_listing_type');
	3009	+ if (!empty($job_types) && !is_wp_error($job_types)) {
	3010	+ $types = array();
	3011	+ foreach ($job_types as $type) {
	3012	+ $types[] = $type->name;
6254	3013	}
	3014	+ $final_content .= "\n\nJob Type: " . implode(', ', $types);
6255	3015	}
6256		-
6257		- // For custom post types like job_listing, include additional fields
6258		- if ($post_type === 'job_listing') {
6259		- // Add job-specific meta if available
6260		- $job_location = get_post_meta($post_id, '_job_location', true);
6261		- if (!empty($job_location)) {
6262		- $final_content .= "\n\nLocation: " . $job_location;
6263		- }
6264		-
6265		- // Get job type terms
6266		- $job_types = get_the_terms($post_id, 'job_listing_type');
6267		- if (!empty($job_types) && !is_wp_error($job_types)) {
6268		- $types = array();
6269		- foreach ($job_types as $type) {
6270		- $types[] = $type->name;
6271		- }
6272		- $final_content .= "\n\nJob Type: " . implode(', ', $types);
6273		- }
6274		-
6275		- // Get company name if available
6276		- $company_name = get_post_meta($post_id, '_company_name', true);
6277		- if (!empty($company_name)) {
6278		- $final_content .= "\n\nCompany: " . $company_name;
6279		- }
6280		- }
6281		-
6282		- // ADD ACF FIELDS SUPPORT (matches ajax_mxchat_process_selected_content behavior)
6283		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
6284		- if (!empty($acf_fields)) {
6285		- $acf_content_parts = array();
6286		- $pdf_attachment_ids = array();
6287		-
6288		- foreach ($acf_fields as $field_name => $field_value) {
6289		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
6290		- if (!empty($formatted_value)) {
6291		- // Convert field name to readable label
6292		- $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name));
6293		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
6294		- }
6295		-
6296		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
6297		- }
6298		-
6299		- if (!empty($acf_content_parts)) {
6300		- $final_content .= "\n\n" . implode("\n", $acf_content_parts);
6301		- }
6302		-
6303		- // Gate the auto-sync PDF-extraction loop behind an opt-in option.
6304		- // Mirrors the per-batch checkbox the manual content selector has; the
6305		- // 25 MB size cap lives in the shared extractor so it applies in both
6306		- // paths regardless. Default OFF — re-parsing every ACF PDF on every
6307		- // editor save is expensive and most sites don't want it.
6308		- $autosync_extract_acf_pdfs = get_option('mxchat_auto_sync_acf_pdfs', '0') === '1';
6309		- if ($autosync_extract_acf_pdfs && !empty($pdf_attachment_ids)) {
6310		- $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
6311		- $pdf_sections = array();
6312		- foreach ($pdf_attachment_ids as $att_id) {
6313		- $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
6314		- if (!empty($pdf_text)) {
6315		- $pdf_title = get_the_title($att_id);
6316		- $pdf_url = wp_get_attachment_url($att_id);
6317		- $header = 'PDF Attachment';
6318		- if (!empty($pdf_title)) {
6319		- $header .= ': ' . $pdf_title;
6320		- }
6321		- if (!empty($pdf_url)) {
6322		- $header .= ' (' . $pdf_url . ')';
6323		- }
6324		- $pdf_sections[] = $header . "\n" . $pdf_text;
6325		- }
6326		- }
6327		- if (!empty($pdf_sections)) {
6328		- $final_content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
6329		- }
6330		- }
6331		- }
6332		-
6333		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
6334		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
6335		- if (!empty($custom_meta)) {
6336		- $meta_content_parts = array();
6337		-
6338		- foreach ($custom_meta as $meta_key => $meta_value) {
6339		- // Convert meta key to readable label
6340		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
6341		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
6342		- }
6343		-
6344		- if (!empty($meta_content_parts)) {
6345		- $final_content .= "\n\n" . implode("\n", $meta_content_parts);
6346		- }
6347		- }
6348		-
6349		- // Embedding decision — custom-provider-aware. Gating on a cloud API key
6350		- // here silently killed auto-sync on keyless custom-embeddings sites,
6351		- // because generate_embedding() routes custom FIRST and never needs the
6352		- // key (plan cbd5fd). Silent-return shape preserved.
6353		- $preflight = MxChat_Utils::embedding_preflight(get_option('mxchat_options'));
6354		- if (!$preflight['ok']) {
6355		- return;
6356		- }
6357		- $api_key = $preflight['api_key'];
6358		-
6359		- // Use the centralized utility function for storage
6360		- $result = MxChat_Utils::submit_content_to_db(
6361		- $final_content,
6362		- $source_url,
6363		- $api_key,
6364		- md5($source_url) // Vector ID for Pinecone
6365		- );
6366	3016
6367		- // After successful storage, apply role restriction based on tags
6368		- if (!is_wp_error($result)) {
6369		- $this->apply_role_restriction_to_post($post_id, $source_url);
	3017	+ // Get company name if available
	3018	+ $company_name = get_post_meta($post_id, '_company_name', true);
	3019	+ if (!empty($company_name)) {
	3020	+ $final_content .= "\n\nCompany: " . $company_name;
6370	3021	}
6371		-}
6372		-
6373		-/**
6374		- * Store the post status and URL before update to detect status transitions
6375		- * This runs before the post is actually updated in the database
6376		- */
6377		-public function mxchat_store_pre_update_status($post_id, $data) {
6378		- // Core is inside wp_insert_post's update branch, so a post_updated WILL fire later
6379		- // this request and can consume the arrival-edge guard (plan a664f3).
6380		- $this->pending_post_update[$post_id] = true;
6381		-
6382		- // Get the current post from database (before update)
6383		- $current_post = get_post($post_id);
	3022	+ }
6384	3023
6385		- if ($current_post) {
6386		- // Store the current status temporarily
6387		- $status_key = 'mxchat_prev_status_' . $post_id;
6388		- set_transient($status_key, $current_post->post_status, HOUR_IN_SECONDS);
6389		-
6390		- // If the post is currently published, also store its URL
6391		- if ($current_post->post_status === 'publish') {
6392		- $url_key = 'mxchat_prev_url_' . $post_id;
6393		- $current_url = get_permalink($post_id);
6394		- set_transient($url_key, $current_url, HOUR_IN_SECONDS);
6395		- }
	3024	+ // Get the source URL
	3025	+ $source_url = get_permalink($post_id);
	3026	+
	3027	+ // Get API key with proper model detection
	3028	+ $options = get_option('mxchat_options');
	3029	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	3030	+
	3031	+ if (strpos($selected_model, 'voyage') === 0) {
	3032	+ $api_key = $options['voyage_api_key'] ?? '';
	3033	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	3034	+ $api_key = $options['gemini_api_key'] ?? '';
	3035	+ } else {
	3036	+ $api_key = $options['api_key'] ?? '';
6396	3037	}
6397		-}
6398		-
6399		-/**
6400		- * Whether auto-sync is enabled for a post type (mirrors the checks used by the
6401		- * update/delete handlers; kept as one helper so new call sites cannot drift).
6402		- */
6403		-private function mxchat_is_auto_sync_enabled($post_type) {
6404		- if ($post_type === 'post') {
6405		- return get_option('mxchat_auto_sync_posts') === '1';
6406		- }
6407		- if ($post_type === 'page') {
6408		- return get_option('mxchat_auto_sync_pages') === '1';
6409		- }
6410		- return get_option('mxchat_auto_sync_' . $post_type) === '1';
6411		-}
6412		-
6413		-/**
6414		- * Remove a post's vectors the moment it leaves 'publish', using the authoritative
6415		- * old status core passes to transition_post_status — no transient involved (plan 816fb1).
6416		- *
6417		- * Covers status changes that never route through wp_update_post (scheduled-expiry
6418		- * plugins and others that flip post_status directly and call wp_transition_post_status),
6419		- * where neither pre_post_update nor post_updated fires and the old detection missed.
6420		- */
6421		-public function mxchat_handle_status_transition($new_status, $old_status, $post) {
6422		- if (!($post instanceof WP_Post) || wp_is_post_revision($post->ID)) {
	3038	+
	3039	+ if (empty($api_key)) {
	3040	+ error_log('MxChat Auto-sync: No API key configured for embedding model');
6423	3041	return;
6424	3042	}
6425		-
6426		- // Arrival edge (plan 3055e1): a post BECOMING published is indexed here, because
6427		- // wp_publish_post() — the path scheduled posts take via check_and_publish_future_post —
6428		- // and direct wp_insert_post(status=publish) creates never fire post_updated, so the
6429		- // auto-sync ADD path alone misses them. Editor publishes also pass through here;
6430		- // the transition_indexed_posts guard keeps mxchat_handle_post_update from embedding
6431		- // a second time in the same request.
6432		- if ($new_status === 'publish' && $old_status !== 'publish') {
6433		- if ($this->mxchat_is_auto_sync_enabled($post->post_type)) {
6434		- $this->mxchat_index_published_post($post->ID, $post);
6435		-
6436		- // Arm the double-fire guard ONLY when a post_updated is actually coming to
6437		- // consume it (plan a664f3). Two publish paths never fire post_updated at all:
6438		- // a direct wp_insert_post(status=publish) create, and wp_publish_post() — the
6439		- // call check_and_publish_future_post() makes for scheduled posts. Arming the
6440		- // guard unconditionally left it set with nothing to consume it, so the NEXT
6441		- // update of that post was swallowed entirely: zero embed calls, no knowledge
6442		- // -base row, silently. Consume-once on this side too, so a guard can never
6443		- // outlive the single save it was armed for.
6444		- if (!empty($this->pending_post_update[$post->ID])) {
6445		- unset($this->pending_post_update[$post->ID]);
6446		- $this->transition_indexed_posts[$post->ID] = true;
6447		- }
6448		- }
6449		- return;
	3043	+
	3044	+ // Use the centralized utility function for storage
	3045	+ $result = MxChat_Utils::submit_content_to_db(
	3046	+ $final_content,
	3047	+ $source_url,
	3048	+ $api_key,
	3049	+ md5($source_url) // Vector ID for Pinecone
	3050	+ );
	3051	+
	3052	+ if (is_wp_error($result)) {
	3053	+ error_log('MxChat Auto-sync failed for post ' . $post_id . ': ' . $result->get_error_message());
6450	3054	}
6451		-
6452		- // Only the publish -> not-publish edge matters here.
6453		- if ($old_status !== 'publish' || $new_status === 'publish') {
6454		- return;
6455		- }
6456		- // Trash is handled by mxchat_handle_post_delete (wp_trash_post) with pre-trash URL
6457		- // resolution; skip to avoid a second network round-trip per trash.
6458		- if ($new_status === 'trash') {
6459		- return;
6460		- }
6461		- if (!$this->mxchat_is_auto_sync_enabled($post->post_type)) {
6462		- return;
6463		- }
6464		-
6465		- $urls = array();
6466		-
6467		- // The DB may already hold the new status when this fires, so get_permalink() on the
6468		- // live post could build a draft-style URL whose md5 misses the stored vector IDs.
6469		- // Reconstruct the published permalink from a clone instead.
6470		- $published_clone = clone $post;
6471		- $published_clone->post_status = 'publish';
6472		- $published_url = get_permalink($published_clone);
6473		- if ($published_url) {
6474		- $urls[] = $published_url;
6475		- }
6476		-
6477		- // Honour the pre-update capture when present (covers a slug change in the same save).
6478		- $previous_url = get_transient('mxchat_prev_url_' . $post->ID);
6479		- if (!empty($previous_url)) {
6480		- $urls[] = $previous_url;
6481		- }
6482		-
6483		- foreach (array_unique($urls) as $url) {
6484		- MxChat_Utils::delete_chunks_for_url($url, 'default');
6485		- }
6486		-
6487		- if (!empty($urls)) {
6488		- $this->transition_deleted_posts[$post->ID] = true;
6489		- }
6490	3055	}
6491	3056
6492		-/**
6493		- * WP-CLI: remove knowledge-base entries left behind by posts that were unpublished,
6494		- * trashed, or made private before the transition_post_status handler existed.
6495		- *
6496		- * Walks every auto-synced post type's non-published posts, reconstructs each one's
6497		- * published-era permalink, and deletes its vectors (routes to Pinecone or the WP table).
6498		- * Deletion is idempotent, so never-indexed posts are a cheap no-op.
6499		- *
6500		- * ## OPTIONS
6501		- *
6502		- * [--dry-run]
6503		- * : Report what would be removed without deleting anything.
6504		- *
6505		- * ## EXAMPLES
6506		- *
6507		- * wp mxchat prune-unpublished --dry-run
6508		- * wp mxchat prune-unpublished
6509		- */
6510		-public function cli_prune_unpublished($args, $assoc_args) {
6511		- global $wpdb;
6512		- $dry_run = !empty($assoc_args['dry-run']);
6513		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
6514	3057
6515		- $candidate_types = array_merge(array('post', 'page'), array_values(get_post_types(array('_builtin' => false), 'names')));
6516		- $synced_types = array();
6517		- foreach ($candidate_types as $type) {
6518		- if ($this->mxchat_is_auto_sync_enabled($type)) {
6519		- $synced_types[] = $type;
6520		- }
6521		- }
6522		- if (empty($synced_types)) {
6523		- WP_CLI::success('No post types have auto-sync enabled; nothing to prune.');
6524		- return;
6525		- }
6526		-
6527		- $scanned = 0;
6528		- $pruned = 0;
6529		- $paged = 1;
6530		- do {
6531		- $query = new WP_Query(array(
6532		- 'post_type' => $synced_types,
6533		- 'post_status' => array('draft', 'pending', 'private', 'future', 'trash'),
6534		- 'posts_per_page' => 100,
6535		- 'paged' => $paged,
6536		- 'fields' => 'ids',
6537		- ));
6538		- foreach ($query->posts as $post_id) {
6539		- $post = get_post($post_id);
6540		- if (!$post) {
6541		- continue;
6542		- }
6543		- $scanned++;
6544		-
6545		- // Rebuild the permalink the post had while published: publish-status clone,
6546		- // with wp_trash_post's __trashed slug suffix stripped for trashed posts.
6547		- $clone = clone $post;
6548		- $clone->post_status = 'publish';
6549		- if (substr($clone->post_name, -9) === '__trashed') {
6550		- $clone->post_name = substr($clone->post_name, 0, -9);
6551		- }
6552		- $url = get_permalink($clone);
6553		- if (!$url) {
6554		- continue;
6555		- }
6556		-
6557		- // Local-table row count is exact in WordPress-DB mode; in Pinecone mode it
6558		- // reads 0 but the delete below still routes to Pinecone and is idempotent.
6559		- $local_rows = (int) $wpdb->get_var($wpdb->prepare(
6560		- "SELECT COUNT(*) FROM {$table} WHERE source_url = %s", $url
6561		- ));
6562		-
6563		- if ($dry_run) {
6564		- if ($local_rows > 0) {
6565		- WP_CLI::log(sprintf('Would remove %d row(s): %s (post %d, %s)', $local_rows, $url, $post_id, $post->post_status));
6566		- $pruned += $local_rows;
6567		- }
6568		- continue;
6569		- }
6570		-
6571		- MxChat_Utils::delete_chunks_for_url($url, 'default');
6572		- if ($local_rows > 0) {
6573		- WP_CLI::log(sprintf('Removed %d row(s): %s (post %d, %s)', $local_rows, $url, $post_id, $post->post_status));
6574		- $pruned += $local_rows;
6575		- }
6576		- }
6577		- $more = $paged < $query->max_num_pages;
6578		- $paged++;
6579		- } while ($more);
6580		-
6581		- WP_CLI::success(sprintf(
6582		- '%s %d local knowledge row(s) across %d non-published post(s) scanned.%s',
6583		- $dry_run ? 'Would remove' : 'Removed',
6584		- $pruned,
6585		- $scanned,
6586		- ' (Pinecone-mode deletions are not counted locally.)'
6587		- ));
6588		-}
6589		-
6590		-/**
6591		- * WP-CLI: repair knowledge-base rows whose PDF text was imported in visual
6592		- * (reversed) order before the RTL normalizer existed. 32bf9e fixed new
6593		- * imports only; this fixes rows already in the table without the customer
6594		- * having to re-source and re-upload the original PDFs (plan d1e6f7).
6595		- *
6596		- * Detection reuses MxChat_Utils::normalize_pdf_rtl() on the stored text: a
6597		- * row is a candidate exactly when the normalizer would change it, so the
6598		- * import-time heuristic and the repair heuristic can never disagree.
6599		- * Repaired rows are RE-EMBEDDED — the stored vector was computed over
6600		- * reversed text and is as broken as the text — so a wet run calls the
6601		- * embedding provider once per repaired row on the site's API key. Runs
6602		- * beyond 25 rows therefore require --yes.
6603		- *
6604		- * Scope notes:
6605		- * - Scans the WordPress knowledge table. Pinecone-mode entries live in
6606		- * Pinecone, not this table, and are not scanned; if a scanned row's bot
6607		- * ALSO has Pinecone enabled (hybrid drift), the repaired entry is
6608		- * re-submitted through the normal import path so the md5-keyed Pinecone
6609		- * vector is replaced too.
6610		- * - Knowledge rows do not carry a bot id; --bot only selects whose
6611		- * embedding configuration (model + key) is used for re-embedding.
6612		- * - The mxchat_pdf_rtl_normalize filter is honoured: a site that disabled
6613		- * normalization gets detections of zero, not surprise rewrites.
6614		- * - The metadata header the PDF importer stores before the text separator
6615		- * is preserved byte-identical; only the text segment is repaired.
6616		- *
6617		- * ## OPTIONS
6618		- *
6619		- * [--dry-run]
6620		- * : List the rows that would be repaired without changing anything.
6621		- *
6622		- * [--bot=<id>]
6623		- * : Embedding configuration to use for re-embedding. Default: default.
6624		- *
6625		- * [--all-content]
6626		- * : Scan every row containing right-to-left text, not just rows with PDF
6627		- * provenance (a page anchor in the source URL, or pdf content type).
6628		- *
6629		- * [--yes]
6630		- * : Proceed even when more than 25 rows need re-embedding (API cost gate).
6631		- *
6632		- * ## EXAMPLES
6633		- *
6634		- * wp mxchat rtl-repair --dry-run
6635		- * wp mxchat rtl-repair
6636		- * wp mxchat rtl-repair --all-content --yes
6637		- */
6638		-public function cli_rtl_repair($args, $assoc_args) {
6639		- global $wpdb;
6640		- $dry_run = !empty($assoc_args['dry-run']);
6641		- $all = !empty($assoc_args['all-content']);
6642		- $yes = !empty($assoc_args['yes']);
6643		- $bot_id = isset($assoc_args['bot']) ? sanitize_key($assoc_args['bot']) : 'default';
6644		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
6645		-
6646		- // Detection pass — no API calls. Walk the table in id batches so a large
6647		- // knowledge base never loads at once.
6648		- $rtl_re = '/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u';
6649		- $candidates = array();
6650		- $scanned = 0;
6651		- $last_id = 0;
6652		- do {
6653		- if ($all) {
6654		- $rows = $wpdb->get_results($wpdb->prepare(
6655		- "SELECT id, article_content, source_url, content_type FROM {$table}
6656		- WHERE id > %d ORDER BY id ASC LIMIT 200",
6657		- $last_id
6658		- ));
6659		- } else {
6660		- $rows = $wpdb->get_results($wpdb->prepare(
6661		- "SELECT id, article_content, source_url, content_type FROM {$table}
6662		- WHERE id > %d AND (source_url LIKE %s OR content_type = 'pdf')
6663		- ORDER BY id ASC LIMIT 200",
6664		- $last_id,
6665		- '%' . $wpdb->esc_like('#page=') . '%'
6666		- ));
6667		- }
6668		- foreach ($rows as $row) {
6669		- $last_id = (int) $row->id;
6670		- $scanned++;
6671		- $content = (string) $row->article_content;
6672		- if (!preg_match($rtl_re, $content)) {
6673		- continue;
6674		- }
6675		- list($header, $text) = $this->mxchat_rtl_repair_split($content);
6676		- $normalized = MxChat_Utils::normalize_pdf_rtl($text, 'rtl-repair row ' . $row->id);
6677		- if (is_string($normalized) && $normalized !== $text) {
6678		- $candidates[] = array(
6679		- 'id' => (int) $row->id,
6680		- 'source_url' => (string) $row->source_url,
6681		- 'content_type' => (string) $row->content_type,
6682		- 'new_content' => $header . $normalized,
6683		- );
6684		- }
6685		- }
6686		- } while (count($rows) === 200);
6687		-
6688		- WP_CLI::log(sprintf('Scanned %d row(s); %d stored in reversed (visual) order.', $scanned, count($candidates)));
6689		- if (empty($candidates)) {
6690		- WP_CLI::success('No reversed RTL rows found — nothing to repair.');
6691		- return;
6692		- }
6693		-
6694		- foreach ($candidates as $c) {
6695		- WP_CLI::log(sprintf('%s row %d %s', $dry_run ? 'Would repair' : 'Will repair', $c['id'], $c['source_url']));
6696		- }
6697		- if ($dry_run) {
6698		- WP_CLI::success(sprintf('Dry run: %d row(s) would be repaired and re-embedded. Run without --dry-run to apply.', count($candidates)));
6699		- return;
6700		- }
6701		-
6702		- // Cost gate: re-embedding spends the customer's API budget.
6703		- WP_CLI::log(sprintf('Re-embedding will call the embedding provider once per row — %d call(s) on this site\'s API key.', count($candidates)));
6704		- if (count($candidates) > 25 && !$yes) {
6705		- WP_CLI::error(sprintf('%d rows need re-embedding (more than 25). Re-run with --yes to confirm the API cost. No rows were changed.', count($candidates)));
6706		- }
6707		-
6708		- $bot_options = $this->get_bot_options($bot_id);
6709		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
6710		- $preflight = MxChat_Utils::embedding_preflight($options);
6711		- if (!$preflight['ok']) {
6712		- WP_CLI::error('Embedding configuration problem: ' . $preflight['reason']);
6713		- }
6714		- $api_key = $preflight['api_key'];
6715		-
6716		- $pinecone_hybrid = $this->mxchat_rtl_repair_pinecone_enabled($bot_id);
6717		- $repaired = 0;
6718		- $failed = 0;
6719		- foreach ($candidates as $c) {
6720		- $vector = MxChat_Utils::regenerate_embedding($c['new_content'], $api_key, $bot_id);
6721		- if (!is_array($vector)) {
6722		- $failed++;
6723		- $reason = is_wp_error($vector) ? $vector->get_error_message() : 'embedding request failed';
6724		- // Text and vector must stay consistent: never write repaired text
6725		- // beside the stale reversed-text vector.
6726		- WP_CLI::warning(sprintf('Row %d NOT repaired — %s. Row left unchanged.', $c['id'], $reason));
6727		- continue;
6728		- }
6729		- $wpdb->update(
6730		- $table,
6731		- array(
6732		- 'article_content' => $c['new_content'],
6733		- 'embedding_vector' => maybe_serialize($vector),
6734		- ),
6735		- array('id' => $c['id']),
6736		- array('%s', '%s'),
6737		- array('%d')
6738		- );
6739		- $repaired++;
6740		- if (class_exists('MxChat_Admin')) {
6741		- MxChat_Admin::mxchat_log_debug('pdf_rtl_repaired', 'Stored KB row restored to logical order and re-embedded', array(
6742		- 'row_id' => $c['id'],
6743		- 'source_url' => $c['source_url'],
6744		- 'bot' => $bot_id,
6745		- ));
6746		- }
6747		- // Hybrid drift: the bot indexes into Pinecone but this row sat in the
6748		- // WP table — push the repaired entry through the normal import path so
6749		- // the md5(source_url)-keyed Pinecone vector is replaced as well.
6750		- if ($pinecone_hybrid) {
6751		- MxChat_Utils::submit_content_to_db(
6752		- $c['new_content'],
6753		- $c['source_url'],
6754		- $api_key,
6755		- null,
6756		- $bot_id,
6757		- $c['content_type'] !== '' ? $c['content_type'] : 'pdf'
6758		- );
6759		- }
6760		- }
6761		-
6762		- WP_CLI::success(sprintf('Repaired + re-embedded %d row(s); %d failed; %d scanned.', $repaired, $failed, $scanned));
6763		-}
6764		-
6765		-/**
6766		- * Split a stored KB row into (metadata header incl. separator, text segment).
6767		- * The PDF importer stores wp_json_encode($metadata) . "\n---\n" . $text —
6768		- * repair must touch only the text and keep the header byte-identical.
6769		- */
6770		-private function mxchat_rtl_repair_split($content) {
6771		- $sep = "\n---\n";
6772		- $pos = strpos($content, $sep);
6773		- if ($pos !== false && $pos > 0 && $content[0] === '{') {
6774		- $maybe_json = substr($content, 0, $pos);
6775		- if (json_decode($maybe_json) !== null) {
6776		- return array(substr($content, 0, $pos + strlen($sep)), substr($content, $pos + strlen($sep)));
6777		- }
6778		- }
6779		- return array('', $content);
6780		-}
6781		-
6782		-/**
6783		- * Mirror of MxChat_Utils::is_pinecone_enabled_for_bot() (private there) for
6784		- * the repair CLI's hybrid-drift check.
6785		- */
6786		-private function mxchat_rtl_repair_pinecone_enabled($bot_id) {
6787		- if ($bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager')) {
6788		- $cfg = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
6789		- if (!empty($cfg)) {
6790		- return !empty($cfg['use_pinecone']) && !empty($cfg['api_key']) && !empty($cfg['host']);
6791		- }
6792		- }
6793		- $po = get_option('mxchat_pinecone_addon_options');
6794		- return !empty($po['mxchat_use_pinecone']) && $po['mxchat_use_pinecone'] !== '0'
6795		- && !empty($po['mxchat_pinecone_api_key']) && !empty($po['mxchat_pinecone_host']);
6796		-}
6797		-
6798	3058	public function mxchat_handle_post_delete($post_id) {
6799	3059	// Get post data before it's deleted
6800	3060	$post = get_post($post_id);
6801	3061
		@@ -6825,51 +3085,85 @@
6825	3085	if (!$should_sync) {
6826	3086	return;
6827	3087	}
6828	3088
6829		- // Resolve the pre-trash URL. wp_trash_post renames the slug with "__trashed" before firing
6830		- // this hook, so get_permalink() here would return the trashed URL and md5() would miss the
6831		- // real vector IDs stored under the original URL.
6832		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
	3089	+ // Get the URL before post is deleted
	3090	+ $source_url = get_permalink($post_id);
6833	3091	if (!$source_url) {
6834		- //error_log('MXChat: Failed to resolve source URL for post ' . $post_id);
	3092	+ error_log('MXChat: Failed to get permalink for post ' . $post_id);
6835	3093	return;
6836	3094	}
6837	3095
6838		- // Use chunk-aware deletion (handles both chunked and non-chunked content)
6839		- $delete_result = MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	3096	+ // Check if Pinecone is enabled
	3097	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	3098	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6840	3099
6841		- if (is_wp_error($delete_result)) {
6842		- //error_log('MXChat: Chunk-aware deletion failed for URL: ' . $source_url . ' - ' . $delete_result->get_error_message());
	3100	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3101	+ // Delete from Pinecone
	3102	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
	3103	+ } else {
	3104	+ // Delete from WordPress DB
	3105	+ global $wpdb;
	3106	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	3107	+
	3108	+ $result = $wpdb->delete(
	3109	+ $table_name,
	3110	+ array('source_url' => $source_url),
	3111	+ array('%s')
	3112	+ );
	3113	+
	3114	+ if ($result === false) {
	3115	+ error_log('MXChat: WordPress DB deletion failed for URL: ' . $source_url);
	3116	+ }
6843	3117	}
	3118	+}
6844	3119
6845		- delete_transient('mxchat_prev_url_' . $post_id);
6846		- delete_transient('mxchat_prev_status_' . $post_id);
6847		-}
6848	3120
6849		-/**
6850		- * Resolve the source URL for a post being trashed/deleted.
6851		- *
6852		- * Why: wp_trash_post appends "__trashed" to the slug before the wp_trash_post action fires, so
6853		- * get_permalink() returns a URL whose md5() won't match the vector IDs stored in Pinecone or
6854		- * the source_url rows in the WP DB. Prefer the URL captured by mxchat_store_pre_update_status
6855		- * (runs on pre_post_update, before the rename); fall back to stripping the __trashed suffix.
6856		- */
6857		-private function mxchat_resolve_pre_trash_url($post_id) {
6858		- $previous_url = get_transient('mxchat_prev_url_' . $post_id);
6859		- if (!empty($previous_url)) {
6860		- return $previous_url;
6861		- }
	3121	+ /**
	3122	+ * Deletes data from Pinecone using a source URL
	3123	+ */
	3124	+ public function mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options) {
	3125	+ $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
	3126	+ $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
6862	3127
6863		- $current = get_permalink($post_id);
6864		- if (!$current) {
6865		- return '';
6866		- }
6867		- return preg_replace('#__trashed(/?)$#', '$1', $current);
6868		-}
	3128	+ if (empty($host) || empty($api_key)) {
	3129	+ //error_log('MXChat: Pinecone deletion failed - missing configuration');
	3130	+ return false;
	3131	+ }
6869	3132
	3133	+ $api_endpoint = "https://{$host}/vectors/delete";
	3134	+ $vector_id = md5($source_url);
6870	3135
	3136	+ $request_body = array(
	3137	+ 'ids' => array($vector_id)
	3138	+ );
6871	3139
	3140	+ $response = wp_remote_post($api_endpoint, array(
	3141	+ 'headers' => array(
	3142	+ 'Api-Key' => $api_key,
	3143	+ 'accept' => 'application/json',
	3144	+ 'content-type' => 'application/json'
	3145	+ ),
	3146	+ 'body' => wp_json_encode($request_body),
	3147	+ 'timeout' => 30
	3148	+ ));
	3149	+
	3150	+ if (is_wp_error($response)) {
	3151	+ //error_log('MXChat: Pinecone deletion error - ' . $response->get_error_message());
	3152	+ return false;
	3153	+ }
	3154	+
	3155	+ $response_code = wp_remote_retrieve_response_code($response);
	3156	+ if ($response_code !== 200) {
	3157	+ //error_log('MXChat: Pinecone deletion failed with status ' . $response_code);
	3158	+ return false;
	3159	+ }
	3160	+
	3161	+ return true;
	3162	+ }
	3163	+
	3164	+
	3165	+
6872	3166	public function mxchat_handle_product_change($post_id, $post, $update) {
6873	3167	if ($post->post_type !== 'product') {
6874	3168	return;
6875	3169	}
		@@ -6891,84 +3185,57 @@
6891	3185	if (!isset($this->options['enable_woocommerce_integration']) ||
6892	3186	!in_array($this->options['enable_woocommerce_integration'], ['1', 'on'])) {
6893	3187	return;
6894	3188	}
6895		-
	3189	+
6896	3190	$source_url = get_permalink($product->get_id());
6897		- $product_id = $product->get_id();
6898		-
	3191	+
6899	3192	// Build product content
6900	3193	$title = $product->get_name();
6901	3194	$description = $product->get_description();
6902	3195	$short_description = $product->get_short_description();
	3196	+ $regular_price = $product->get_regular_price();
	3197	+ $sale_price = $product->get_sale_price();
6903	3198	$sku = $product->get_sku();
6904		-
	3199	+
6905	3200	// Format content consistently
6906	3201	$content = $title . "\n\n";
6907		-
	3202	+
	3203	+ if (!empty($description)) {
	3204	+ $content .= wp_strip_all_tags($description) . "\n\n";
	3205	+ }
	3206	+
6908	3207	if (!empty($short_description)) {
6909	3208	$content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
6910	3209	}
6911		-
6912		- if (!empty($description)) {
6913		- $content .= wp_strip_all_tags($description) . "\n\n";
	3210	+
	3211	+ $content .= "Price: $" . $regular_price . "\n";
	3212	+
	3213	+ if (!empty($sale_price)) {
	3214	+ $content .= "Sale Price: $" . $sale_price . "\n";
6914	3215	}
6915		-
6916		- // Add pricing information (base-currency pinned — see mxchat_product_price_lines)
6917		- $content .= $this->mxchat_product_price_lines($product);
6918		-
	3216	+
6919	3217	if (!empty($sku)) {
6920	3218	$content .= "SKU: " . $sku . "\n";
6921	3219	}
6922		-
6923		- // Get product categories
6924		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
6925		- if (!empty($categories) && !is_wp_error($categories)) {
6926		- $content .= "Categories: " . implode(', ', $categories) . "\n";
	3220	+
	3221	+ // Get API key with proper model detection
	3222	+ $options = get_option('mxchat_options');
	3223	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	3224	+
	3225	+ if (strpos($selected_model, 'voyage') === 0) {
	3226	+ $api_key = $options['voyage_api_key'] ?? '';
	3227	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	3228	+ $api_key = $options['gemini_api_key'] ?? '';
	3229	+ } else {
	3230	+ $api_key = $options['api_key'] ?? '';
6927	3231	}
6928		-
6929		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
6930		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
6931		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
6932		- foreach ($custom_tabs as $tab) {
6933		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
6934		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
6935		-
6936		- if (!empty($tab_title) && !empty($tab_content)) {
6937		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
6938		- }
6939		- }
6940		- }
6941		-
6942		- // Also check for reusable/saved tabs applied to this product
6943		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
6944		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
6945		- // Get the saved tabs option
6946		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
6947		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
6948		- foreach ($applied_saved_tabs as $saved_tab_id) {
6949		- if (isset($saved_tabs[$saved_tab_id])) {
6950		- $tab = $saved_tabs[$saved_tab_id];
6951		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
6952		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
6953		-
6954		- if (!empty($tab_title) && !empty($tab_content)) {
6955		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
6956		- }
6957		- }
6958		- }
6959		- }
6960		- }
6961		-
6962		- // Embedding decision — custom-provider-aware (plan cbd5fd); silent-return
6963		- // shape preserved.
6964		- $preflight = MxChat_Utils::embedding_preflight(get_option('mxchat_options'));
6965		- if (!$preflight['ok']) {
6966		- //error_log('MxChat Auto-sync: embedding pre-flight failed: ' . $preflight['reason']);
	3232	+
	3233	+ if (empty($api_key)) {
	3234	+ error_log('MxChat Auto-sync: No API key configured for embedding model');
6967	3235	return;
6968	3236	}
6969		- $api_key = $preflight['api_key'];
6970		-
	3237	+
6971	3238	// Use the centralized utility function for storage
6972	3239	$result = MxChat_Utils::submit_content_to_db(
6973	3240	$content,
6974	3241	$source_url,
		@@ -6974,16 +3241,11 @@
6974	3241	$source_url,
6975	3242	$api_key,
6976	3243	md5($source_url) // Vector ID for Pinecone
6977	3244	);
6978		-
6979		- // After successful storage, apply role restriction based on tags
6980		- if (!is_wp_error($result)) {
6981		- $this->apply_role_restriction_to_post($product_id, $source_url);
6982		- }
6983		-
	3245	+
6984	3246	if (is_wp_error($result)) {
6985		- //error_log('MxChat WooCommerce sync failed for product ' . $product_id . ': ' . $result->get_error_message());
	3247	+ error_log('MxChat WooCommerce sync failed for product ' . $product->get_id() . ': ' . $result->get_error_message());
6986	3248	}
6987	3249	}
6988	3250
6989	3251	public function mxchat_handle_product_delete($post_id) {
		@@ -6990,1133 +3252,30 @@
6990	3252	if (get_post_type($post_id) !== 'product') {
6991	3253	return;
6992	3254	}
6993	3255
6994		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
6995		- if (!$source_url) {
6996		- return;
6997		- }
	3256	+ $source_url = get_permalink($post_id);
6998	3257
6999		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
7000		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
7001		-
7002		- delete_transient('mxchat_prev_url_' . $post_id);
7003		- delete_transient('mxchat_prev_status_' . $post_id);
7004		-}
7005		-
7006		-/**
7007		- * Handle individual Pinecone content deletion
7008		- */
7009		-public function mxchat_handle_pinecone_prompt_delete() {
7010		- // Check permissions
7011		- if (!current_user_can('manage_options')) {
7012		- wp_die(esc_html__('You do not have sufficient permissions.', 'mxchat'));
7013		- }
7014		-
7015		- // Verify nonce
7016		- if (!isset($_GET['_wpnonce']) || !wp_verify_nonce($_GET['_wpnonce'], 'mxchat_delete_pinecone_prompt_nonce')) {
7017		- wp_die(esc_html__('Security check failed.', 'mxchat'));
7018		- }
7019		-
7020		- $vector_id = isset($_GET['vector_id']) ? sanitize_text_field($_GET['vector_id']) : '';
7021		-
7022		- if (empty($vector_id)) {
7023		- set_transient('mxchat_admin_notice_error',
7024		- esc_html__('Invalid vector ID.', 'mxchat'),
7025		- 30
7026		- );
7027		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
7028		- exit;
7029		- }
7030		-
7031		- // Get Pinecone settings
	3258	+ // Check if Pinecone is enabled
7032	3259	$pinecone_options = get_option('mxchat_pinecone_addon_options', array());
7033	3260	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7034		-
7035		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
7036		- set_transient('mxchat_admin_notice_error',
7037		- esc_html__('Pinecone is not properly configured.', 'mxchat'),
7038		- 30
7039		- );
7040		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
7041		- exit;
7042		- }
7043		-
7044		- // Delete from Pinecone
7045		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
7046		- $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
7047		- $vector_id,
7048		- $pinecone_options['mxchat_pinecone_api_key'],
7049		- $pinecone_options['mxchat_pinecone_host'],
7050		- $pinecone_options['mxchat_pinecone_namespace'] ?? ''
7051		- );
7052		-
7053		- if ($result['success']) {
7054		- // No cache clearing needed since we removed caching
7055		- set_transient('mxchat_admin_notice_success',
7056		- esc_html__('Entry deleted successfully from Pinecone.', 'mxchat'),
7057		- 30
7058		- );
7059		- } else {
7060		- set_transient('mxchat_admin_notice_error',
7061		- esc_html__('Failed to delete entry: ', 'mxchat') . $result['message'],
7062		- 30
7063		- );
7064		- }
7065		-
7066		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
7067		- exit;
7068		-}
7069		-/**
7070		- * Handle individual Pinecone content deletion via AJAX
7071		- */
7072		-public function ajax_mxchat_delete_pinecone_prompt() {
7073		- // Verify nonce and permissions
7074		- if (!check_ajax_referer('mxchat_delete_pinecone_prompt_nonce', 'nonce', false)) {
7075		- wp_send_json_error('Invalid nonce');
7076		- exit;
7077		- }
7078		-
7079		- if (!current_user_can('manage_options')) {
7080		- wp_send_json_error('Unauthorized access');
7081		- exit;
7082		- }
7083		-
7084		- $vector_id = isset($_POST['vector_id']) ? sanitize_text_field($_POST['vector_id']) : '';
7085		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
7086		-
7087		- if (empty($vector_id)) {
7088		- wp_send_json_error('Missing vector ID');
7089		- exit;
7090		- }
7091		-
7092		- // Get bot-specific Pinecone settings
7093		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
7094		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
7095		-
7096		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7097		-
7098		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
7099		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
7100		- exit;
7101		- }
7102		-
7103		- // Delete from the correct Pinecone index and namespace
7104		- $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
7105		- $vector_id,
7106		- $pinecone_options['mxchat_pinecone_api_key'],
7107		- $pinecone_options['mxchat_pinecone_host'],
7108		- $pinecone_options['mxchat_pinecone_namespace'] ?? ''
7109		- );
7110		-
7111		- if ($result['success']) {
7112		- // No cache clearing needed since we removed caching
7113		- wp_send_json_success(array(
7114		- 'message' => 'Entry deleted successfully from Pinecone',
7115		- 'vector_id' => $vector_id,
7116		- 'bot_id' => $bot_id
7117		- ));
7118		- } else {
7119		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete from Pinecone: ' . $result['message'], array('vector_id' => $vector_id, 'bot_id' => $bot_id));
7120		- wp_send_json_error('Failed to delete from Pinecone: ' . $result['message']);
7121		- }
7122		-
7123		- exit;
7124		-}
7125	3261
7126		-/**
7127		- * Handle deletion of all chunks for a given source URL via AJAX
7128		- * Follows the same pattern as ajax_mxchat_delete_pinecone_prompt
7129		- */
7130		-public function ajax_mxchat_delete_chunks_by_url() {
7131		- // Verify nonce and permissions
7132		- if (!check_ajax_referer('mxchat_delete_chunks_nonce', 'nonce', false)) {
7133		- wp_send_json_error('Invalid nonce');
7134		- exit;
7135		- }
7136		-
7137		- if (!current_user_can('manage_options')) {
7138		- wp_send_json_error('Unauthorized access');
7139		- exit;
7140		- }
7141		-
7142		- $source_url = isset($_POST['source_url']) ? esc_url_raw($_POST['source_url']) : '';
7143		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
7144		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
7145		-
7146		- if (empty($source_url)) {
7147		- wp_send_json_error('Missing source URL');
7148		- exit;
7149		- }
7150		-
7151		- // Generate the base vector ID from the source URL (same as how chunks are created)
7152		- $base_vector_id = md5($source_url);
7153		-
7154		- if ($data_source === 'pinecone') {
7155		- // Get bot-specific Pinecone settings (same as working delete function)
7156		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
7157		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
7158		-
7159		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7160		-
7161		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
7162		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
7163		- exit;
7164		- }
7165		-
7166		- $api_key = $pinecone_options['mxchat_pinecone_api_key'];
7167		- $host = $pinecone_options['mxchat_pinecone_host'];
7168		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
7169		-
7170		- // Collect all vector IDs to delete
7171		- $vectors_to_delete = array();
7172		-
7173		- // Add the original single-vector ID (for non-chunked content)
7174		- $vectors_to_delete[] = $base_vector_id;
7175		-
7176		- // Use Pinecone list API to find all chunk vectors with this prefix
7177		- // NOTE: Pinecone List API is a GET request with query parameters, not POST
7178		- $prefix = $base_vector_id . '_chunk_';
7179		-
7180		- $query_params = array(
7181		- 'prefix' => $prefix,
7182		- 'limit' => 100
7183		- );
7184		-
7185		- if (!empty($namespace)) {
7186		- $query_params['namespace'] = $namespace;
7187		- }
7188		-
7189		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
7190		-
7191		- $list_response = wp_remote_get($list_url, array(
7192		- 'headers' => array(
7193		- 'Api-Key' => $api_key,
7194		- 'accept' => 'application/json'
7195		- ),
7196		- 'timeout' => 30
7197		- ));
7198		-
7199		- if (!is_wp_error($list_response)) {
7200		- $list_body_response = wp_remote_retrieve_body($list_response);
7201		- $list_data = json_decode($list_body_response, true);
7202		- if (!empty($list_data['vectors'])) {
7203		- foreach ($list_data['vectors'] as $vector) {
7204		- if (isset($vector['id'])) {
7205		- $vectors_to_delete[] = $vector['id'];
7206		- }
7207		- }
7208		- }
7209		- }
7210		-
7211		- if (empty($vectors_to_delete)) {
7212		- wp_send_json_success(array(
7213		- 'message' => 'No vectors found to delete',
7214		- 'source_url' => $source_url
7215		- ));
7216		- exit;
7217		- }
7218		-
7219		- // Delete all vectors using the same endpoint as the working function
7220		- $delete_url = "https://{$host}/vectors/delete";
7221		-
7222		- $delete_body = array(
7223		- 'ids' => $vectors_to_delete
7224		- );
7225		-
7226		- if (!empty($namespace)) {
7227		- $delete_body['namespace'] = $namespace;
7228		- }
7229		-
7230		- $delete_response = wp_remote_post($delete_url, array(
7231		- 'headers' => array(
7232		- 'Api-Key' => $api_key,
7233		- 'accept' => 'application/json',
7234		- 'content-type' => 'application/json'
7235		- ),
7236		- 'body' => wp_json_encode($delete_body),
7237		- 'timeout' => 30
7238		- ));
7239		-
7240		- if (is_wp_error($delete_response)) {
7241		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete chunks from Pinecone: ' . $delete_response->get_error_message(), array('source_url' => $source_url));
7242		- wp_send_json_error('Failed to delete from Pinecone: ' . $delete_response->get_error_message());
7243		- exit;
7244		- }
7245		-
7246		- $response_code = wp_remote_retrieve_response_code($delete_response);
7247		-
7248		- if ($response_code !== 200) {
7249		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Pinecone API error (HTTP ' . $response_code . ')', array('source_url' => $source_url));
7250		- wp_send_json_error('Pinecone API error (HTTP ' . $response_code . ')');
7251		- exit;
7252		- }
7253		-
7254		- wp_send_json_success(array(
7255		- 'message' => 'All chunks deleted successfully from Pinecone',
7256		- 'source_url' => $source_url,
7257		- 'deleted_count' => count($vectors_to_delete)
7258		- ));
7259		-
	3262	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3263	+ // Delete from Pinecone
	3264	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
7260	3265	} else {
7261		- // WordPress database deletion
	3266	+ // Delete from WordPress DB
7262	3267	global $wpdb;
7263	3268	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7264		-
7265		- $result = $wpdb->delete(
7266		- $table_name,
7267		- array('source_url' => $source_url),
7268		- array('%s')
7269		- );
7270		-
7271		- if ($result === false) {
7272		- MxChat_Admin::mxchat_log_debug('database_error', 'Failed to delete from database: ' . $wpdb->last_error, array('source_url' => $source_url));
7273		- wp_send_json_error('Failed to delete from database: ' . $wpdb->last_error);
7274		- exit;
7275		- }
7276		-
7277		- wp_send_json_success(array(
7278		- 'message' => 'All chunks deleted successfully from database',
7279		- 'source_url' => $source_url,
7280		- 'deleted_count' => $result
7281		- ));
7282		- }
7283		-
7284		- exit;
7285		-}
7286		-
7287		-/**
7288		- * Handle individual WordPress database content deletion via AJAX
7289		- * Mirrors the Pinecone delete handler but for WordPress database entries
7290		- */
7291		-public function ajax_mxchat_delete_wordpress_prompt() {
7292		- // Verify nonce and permissions
7293		- if (!check_ajax_referer('mxchat_delete_wordpress_prompt_nonce', 'nonce', false)) {
7294		- wp_send_json_error('Invalid nonce');
7295		- exit;
7296		- }
7297		-
7298		- if (!current_user_can('manage_options')) {
7299		- wp_send_json_error('Unauthorized access');
7300		- exit;
7301		- }
7302		-
7303		- $entry_id = isset($_POST['entry_id']) ? intval($_POST['entry_id']) : 0;
7304		-
7305		- if (empty($entry_id)) {
7306		- wp_send_json_error('Missing entry ID');
7307		- exit;
7308		- }
7309		-
7310		- global $wpdb;
7311		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7312		-
7313		- // Clear cache for this entry
7314		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
7315		-
7316		- // Delete from database
7317		- $result = $wpdb->delete(
7318		- $table_name,
7319		- array('id' => $entry_id),
7320		- array('%d')
7321		- );
7322		-
7323		- if ($result !== false) {
7324		- wp_send_json_success(array(
7325		- 'message' => 'Entry deleted successfully',
7326		- 'entry_id' => $entry_id
7327		- ));
7328		- } else {
7329		- wp_send_json_error('Failed to delete entry from database');
7330		- }
7331		-
7332		- exit;
7333		-}
7334		-
7335		-/**
7336		- * Handle bulk deletion of knowledge entries via AJAX
7337		- * Supports both Pinecone and WordPress database entries
7338		- */
7339		-public function ajax_mxchat_bulk_delete_knowledge() {
7340		- // Verify nonce and permissions
7341		- if (!check_ajax_referer('mxchat_bulk_delete_knowledge_nonce', 'nonce', false)) {
7342		- wp_send_json_error('Invalid nonce');
7343		- exit;
7344		- }
7345		-
7346		- if (!current_user_can('manage_options')) {
7347		- wp_send_json_error('Unauthorized access');
7348		- exit;
7349		- }
7350		-
7351		- $entries = isset($_POST['entries']) ? $_POST['entries'] : array();
7352		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
7353		-
7354		- if (empty($entries) || !is_array($entries)) {
7355		- wp_send_json_error('No entries provided');
7356		- exit;
7357		- }
7358		-
7359		- // Extend execution time — bulk Pinecone operations can take a while
7360		- if (function_exists('set_time_limit')) {
7361		- set_time_limit(120);
7362		- }
7363		-
7364		- $success_ids = array();
7365		- $failed_ids = array();
7366		- $errors = array();
7367		-
7368		- global $wpdb;
7369		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7370		-
7371		- // Get Pinecone manager for Pinecone deletions
7372		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
7373		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
7374		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7375		-
7376		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
7377		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
7378		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
7379		-
7380		- // =============================================
7381		- // PHASE 1: Collect all Pinecone vector IDs
7382		- // and separate WordPress entries
7383		- // =============================================
7384		- $pinecone_entry_ids = array(); // entry IDs that are pinecone-sourced
7385		- $wordpress_entries = array(); // entries for WordPress DB deletion
7386		- $all_vector_ids = array(); // all pinecone vector IDs to delete in one batch
7387		-
7388		- foreach ($entries as $entry) {
7389		- $entry_id = sanitize_text_field($entry['id'] ?? '');
7390		- $source = sanitize_text_field($entry['source'] ?? 'wordpress');
7391		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
7392		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
7393		-
7394		- if (empty($entry_id)) {
7395		- continue;
7396		- }
7397		-
7398		- if ($source === 'pinecone') {
7399		- if (!$use_pinecone || empty($api_key)) {
7400		- $failed_ids[] = $entry_id;
7401		- $errors[] = "Pinecone not configured for entry: $entry_id";
7402		- continue;
7403		- }
7404		-
7405		- $pinecone_entry_ids[] = $entry_id;
7406		-
7407		- if ($is_group && !empty($source_url)) {
7408		- // Grouped/chunked entry: collect base ID + chunk IDs via List API
7409		- $base_vector_id = md5($source_url);
7410		- $all_vector_ids[] = $base_vector_id;
7411		-
7412		- $list_url = 'https://' . $host . '/vectors/list?prefix=' . urlencode($base_vector_id . '_chunk_') . '&limit=100';
7413		- if (!empty($namespace)) {
7414		- $list_url .= '&namespace=' . rawurlencode($namespace);
7415		- }
7416		- $list_response = wp_remote_get($list_url, array(
7417		- 'headers' => array(
7418		- 'Api-Key' => $api_key,
7419		- 'accept' => 'application/json'
7420		- ),
7421		- 'timeout' => 30
7422		- ));
7423		-
7424		- if (!is_wp_error($list_response)) {
7425		- $list_body = json_decode(wp_remote_retrieve_body($list_response), true);
7426		- if (!empty($list_body['vectors']) && is_array($list_body['vectors'])) {
7427		- foreach ($list_body['vectors'] as $vector) {
7428		- if (isset($vector['id'])) {
7429		- $all_vector_ids[] = $vector['id'];
7430		- }
7431		- }
7432		- }
7433		- }
7434		- } else {
7435		- // Single entry: the entry_id IS the vector ID
7436		- $all_vector_ids[] = $entry_id;
7437		- }
7438		- } else {
7439		- $wordpress_entries[] = $entry;
7440		- }
7441		- }
7442		-
7443		- // =============================================
7444		- // PHASE 2: Single batch delete to Pinecone
7445		- // =============================================
7446		- if (!empty($all_vector_ids)) {
7447		- $all_vector_ids = array_values(array_unique($all_vector_ids));
7448		- $pinecone_success = true;
7449		- $batches = array_chunk($all_vector_ids, 100);
7450		-
7451		- foreach ($batches as $batch) {
7452		- $delete_body = array('ids' => $batch);
7453		- if (!empty($namespace)) {
7454		- $delete_body['namespace'] = $namespace;
7455		- }
7456		- $delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
7457		- 'headers' => array(
7458		- 'Api-Key' => $api_key,
7459		- 'accept' => 'application/json',
7460		- 'content-type' => 'application/json'
7461		- ),
7462		- 'body' => wp_json_encode($delete_body),
7463		- 'timeout' => 60
7464		- ));
7465		-
7466		- if (is_wp_error($delete_response)) {
7467		- $pinecone_success = false;
7468		- $errors[] = 'Pinecone batch deletion failed: ' . $delete_response->get_error_message();
7469		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed: ' . $delete_response->get_error_message());
7470		- } else {
7471		- $response_code = wp_remote_retrieve_response_code($delete_response);
7472		- if ($response_code !== 200) {
7473		- $pinecone_success = false;
7474		- $response_body = wp_remote_retrieve_body($delete_response);
7475		- $errors[] = "Pinecone API error (HTTP $response_code)";
7476		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed (HTTP ' . $response_code . ')', array('response' => substr($response_body, 0, 200)));
7477		- }
7478		- }
7479		- }
7480		-
7481		- // Mark all pinecone entries based on batch result
7482		- foreach ($pinecone_entry_ids as $eid) {
7483		- if ($pinecone_success) {
7484		- $success_ids[] = $eid;
7485		- } else {
7486		- $failed_ids[] = $eid;
7487		- }
7488		- }
7489		- }
7490		-
7491		- // =============================================
7492		- // PHASE 3: WordPress database deletions
7493		- // =============================================
7494		- foreach ($wordpress_entries as $entry) {
7495		- $entry_id = sanitize_text_field($entry['id'] ?? '');
7496		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
7497		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
7498		-
7499		- if (empty($entry_id)) {
7500		- continue;
7501		- }
7502		-
7503		- try {
7504		- if ($is_group && !empty($source_url)) {
7505		- $result = $wpdb->delete(
7506		- $table_name,
7507		- array('source_url' => $source_url),
7508		- array('%s')
7509		- );
7510		- } else {
7511		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
7512		- $result = $wpdb->delete(
7513		- $table_name,
7514		- array('id' => intval($entry_id)),
7515		- array('%d')
7516		- );
7517		- }
7518		-
7519		- if ($result !== false) {
7520		- $success_ids[] = $entry_id;
7521		- } else {
7522		- $failed_ids[] = $entry_id;
7523		- $errors[] = "Database error for entry: $entry_id";
7524		- }
7525		- } catch (Exception $e) {
7526		- $failed_ids[] = $entry_id;
7527		- $errors[] = $e->getMessage();
7528		- }
7529		- }
7530		-
7531		- wp_send_json_success(array(
7532		- 'success_ids' => $success_ids,
7533		- 'failed_ids' => $failed_ids,
7534		- 'errors' => $errors,
7535		- 'total_processed' => count($success_ids) + count($failed_ids)
7536		- ));
7537		-
7538		- exit;
7539		-}
7540		-
7541		-/**
7542		- * Get hierarchical roles for dropdown
7543		- */
7544		-public function mxchat_get_role_options() {
7545		- return array(
7546		- 'public' => __('Public (Everyone)', 'mxchat'),
7547		- 'logged_in' => __('Logged In Users', 'mxchat'),
7548		- 'subscriber' => __('Subscribers & Above', 'mxchat'),
7549		- 'contributor' => __('Contributors & Above', 'mxchat'),
7550		- 'author' => __('Authors & Above', 'mxchat'),
7551		- 'editor' => __('Editors & Above', 'mxchat'),
7552		- 'administrator' => __('Administrators Only', 'mxchat')
7553		- );
7554		-}
7555		-
7556		-/**
7557		- * Check if user has access to content based on role restriction
7558		- */
7559		-public function mxchat_user_has_content_access($role_restriction) {
7560		- // Public content is always accessible
7561		- if ($role_restriction === 'public' || empty($role_restriction)) {
7562		- return true;
7563		- }
7564		-
7565		- // Check if user is logged in for logged_in restriction
7566		- if ($role_restriction === 'logged_in') {
7567		- return is_user_logged_in();
7568		- }
7569		-
7570		- // If not logged in, no access to role-restricted content
7571		- if (!is_user_logged_in()) {
7572		- return false;
7573		- }
7574		-
7575		- $user = wp_get_current_user();
7576		- $user_roles = $user->roles;
7577		-
7578		- if (empty($user_roles)) {
7579		- return false;
7580		- }
7581		-
7582		- // Define role hierarchy (higher number = higher access)
7583		- $hierarchy = array(
7584		- 'subscriber' => 1,
7585		- 'contributor' => 2,
7586		- 'author' => 3,
7587		- 'editor' => 4,
7588		- 'administrator' => 5
7589		- );
7590		-
7591		- // Get required level
7592		- $required_level = isset($hierarchy[$role_restriction]) ? $hierarchy[$role_restriction] : 0;
7593		-
7594		- // Check if user has required level or higher
7595		- foreach ($user_roles as $user_role) {
7596		- $user_level = isset($hierarchy[$user_role]) ? $hierarchy[$user_role] : 0;
7597		- if ($user_level >= $required_level) {
7598		- return true;
7599		- }
7600		- }
7601		-
7602		- return false;
7603		-}
7604		-
7605		-/**
7606		- * Handle role restriction updates via AJAX
7607		- * Removed cache clearing call since we removed caching
7608		- */
7609		-public function ajax_mxchat_update_role_restriction() {
7610		- // Verify nonce and permissions
7611		- if (!check_ajax_referer('mxchat_update_role_nonce', 'nonce', false)) {
7612		- wp_send_json_error('Invalid nonce');
7613		- exit;
7614		- }
7615		-
7616		- if (!current_user_can('manage_options')) {
7617		- wp_send_json_error('Unauthorized access');
7618		- exit;
7619		- }
7620		-
7621		- $entry_id = isset($_POST['entry_id']) ? sanitize_text_field($_POST['entry_id']) : '';
7622		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
7623		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
7624		-
7625		- if (empty($entry_id)) {
7626		- wp_send_json_error('Invalid entry ID');
7627		- exit;
7628		- }
7629		-
7630		- // Get knowledge manager instance to validate role restriction
7631		- $knowledge_manager = MxChat_Knowledge_Manager::get_instance();
7632		- $valid_roles = array_keys($knowledge_manager->mxchat_get_role_options());
7633		- if (!in_array($role_restriction, $valid_roles)) {
7634		- wp_send_json_error('Invalid role restriction');
7635		- exit;
7636		- }
7637		-
7638		- global $wpdb;
7639		-
7640		- if ($data_source === 'pinecone') {
7641		- // Handle Pinecone role restriction (stored separately in WordPress table)
7642		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7643	3269
7644		- // Use REPLACE to insert or update the role restriction
7645		- $result = $wpdb->replace(
7646		- $roles_table,
7647		- array(
7648		- 'vector_id' => $entry_id,
7649		- 'role_restriction' => $role_restriction,
7650		- 'updated_at' => current_time('mysql')
7651		- ),
7652		- array('%s', '%s', '%s')
7653		- );
7654		-
7655		- // No cache clearing needed since we removed caching
7656		-
7657		- } else {
7658		- // Handle WordPress database role restriction (existing functionality)
7659		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7660		-
7661		- $result = $wpdb->update(
	3270	+ $wpdb->delete(
7662	3271	$table_name,
7663		- array('role_restriction' => $role_restriction),
7664		- array('id' => absint($entry_id)),
7665		- array('%s'),
7666		- array('%d')
7667		- );
7668		- }
7669		-
7670		- if ($result === false) {
7671		- wp_send_json_error('Database update failed: ' . $wpdb->last_error);
7672		- exit;
7673		- }
7674		-
7675		- wp_send_json_success(array(
7676		- 'message' => 'Role restriction updated successfully',
7677		- 'role_restriction' => $role_restriction,
7678		- 'data_source' => $data_source,
7679		- 'entry_id' => $entry_id
7680		- ));
7681		- exit;
7682		-}
7683		-
7684		-// ========================================
7685		-// ROLE-BASED CONTENT RESTRICTIONS - NEW FUNCTIONS
7686		-// Add these to your MxChat_Knowledge_Manager class
7687		-// ========================================
7688		-
7689		-/**
7690		- * Initialize role-based content hooks
7691		- * Add this call to your __construct() or mxchat_init_hooks() method
7692		- */
7693		-private function mxchat_init_role_hooks() {
7694		- // AJAX handlers for tag-role mappings
7695		- add_action('wp_ajax_mxchat_add_tag_role_mapping', array($this, 'ajax_add_tag_role_mapping'));
7696		- add_action('wp_ajax_mxchat_delete_tag_role_mapping', array($this, 'ajax_delete_tag_role_mapping'));
7697		- add_action('wp_ajax_mxchat_get_tag_role_mappings', array($this, 'ajax_get_tag_role_mappings'));
7698		- add_action('wp_ajax_mxchat_bulk_update_tag_roles', array($this, 'ajax_bulk_update_tag_roles'));
7699		-
7700		- // Hook to automatically update role restrictions when tags are added/removed
7701		- add_action('set_object_terms', array($this, 'handle_tag_change'), 10, 6);
7702		-
7703		- // Hook to apply role restrictions on auto-sync
7704		- add_action('mxchat_content_stored', array($this, 'apply_role_restriction_after_storage'), 10, 2);
7705		-}
7706		-
7707		-/**
7708		- * Add tag-role mapping via AJAX
7709		- */
7710		-public function ajax_add_tag_role_mapping() {
7711		- // Verify nonce and permissions
7712		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7713		-
7714		- if (!current_user_can('manage_options')) {
7715		- wp_send_json_error('Unauthorized access');
7716		- exit;
7717		- }
7718		-
7719		- $tag_input = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
7720		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
7721		-
7722		- if (empty($tag_input)) {
7723		- wp_send_json_error('Please enter a tag name or slug');
7724		- exit;
7725		- }
7726		-
7727		- // Validate role restriction
7728		- $valid_roles = array_keys($this->mxchat_get_role_options());
7729		- if (!in_array($role_restriction, $valid_roles)) {
7730		- wp_send_json_error('Invalid role restriction');
7731		- exit;
7732		- }
7733		-
7734		- // Resolve the tag by slug first, then fall back to its display name, so users can
7735		- // enter either "premium-content" or "Premium Content". (plan b8bcf5 — the field is
7736		- // labeled by name but previously validated by slug only, producing the confusing
7737		- // "Tag does not exist in WordPress" error when a real tag's name was typed.)
7738		- $term = get_term_by('slug', $tag_input, 'post_tag');
7739		- if (!$term) {
7740		- $term = get_term_by('name', $tag_input, 'post_tag');
7741		- }
7742		- if (!$term) {
7743		- wp_send_json_error('No tag with that name or slug exists yet. Create it under Posts → Tags first, then enter its name or slug.');
7744		- exit;
7745		- }
7746		-
7747		- // Always key the mapping by the RESOLVED slug — apply_role_restriction_to_post()
7748		- // compares against each post's tag slugs, so the stored key must be a slug,
7749		- // never the raw (possibly display-name) input.
7750		- $tag_slug = $term->slug;
7751		-
7752		- // Get existing mappings
7753		- $mappings = get_option('mxchat_tag_role_mappings', array());
7754		-
7755		- // Check if mapping already exists
7756		- if (isset($mappings[$tag_slug])) {
7757		- wp_send_json_error('Mapping for this tag already exists');
7758		- exit;
7759		- }
7760		-
7761		- // Add new mapping
7762		- $mappings[$tag_slug] = $role_restriction;
7763		- update_option('mxchat_tag_role_mappings', $mappings);
7764		-
7765		- wp_send_json_success(array(
7766		- 'message' => 'Tag-role mapping added successfully',
7767		- 'tag_slug' => $tag_slug,
7768		- 'role_restriction' => $role_restriction
7769		- ));
7770		- exit;
7771		-}
7772		-
7773		-/**
7774		- * Delete tag-role mapping via AJAX
7775		- */
7776		-public function ajax_delete_tag_role_mapping() {
7777		- // Verify nonce and permissions
7778		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7779		-
7780		- if (!current_user_can('manage_options')) {
7781		- wp_send_json_error('Unauthorized access');
7782		- exit;
7783		- }
7784		-
7785		- $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
7786		-
7787		- if (empty($tag_slug)) {
7788		- wp_send_json_error('Tag slug is required');
7789		- exit;
7790		- }
7791		-
7792		- // Get existing mappings
7793		- $mappings = get_option('mxchat_tag_role_mappings', array());
7794		-
7795		- // Check if mapping exists
7796		- if (!isset($mappings[$tag_slug])) {
7797		- wp_send_json_error('Mapping does not exist');
7798		- exit;
7799		- }
7800		-
7801		- // Remove mapping
7802		- unset($mappings[$tag_slug]);
7803		- update_option('mxchat_tag_role_mappings', $mappings);
7804		-
7805		- wp_send_json_success(array(
7806		- 'message' => 'Tag-role mapping deleted successfully',
7807		- 'tag_slug' => $tag_slug
7808		- ));
7809		- exit;
7810		-}
7811		-
7812		-/**
7813		- * Get all tag-role mappings via AJAX
7814		- */
7815		-public function ajax_get_tag_role_mappings() {
7816		- // Verify nonce and permissions
7817		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7818		-
7819		- if (!current_user_can('manage_options')) {
7820		- wp_send_json_error('Unauthorized access');
7821		- exit;
7822		- }
7823		-
7824		- // Get mappings
7825		- $mappings = get_option('mxchat_tag_role_mappings', array());
7826		- $role_options = $this->mxchat_get_role_options();
7827		-
7828		- $formatted_mappings = array();
7829		-
7830		- foreach ($mappings as $tag_slug => $role_restriction) {
7831		- // Get tag object
7832		- $term = get_term_by('slug', $tag_slug, 'post_tag');
7833		-
7834		- // Count posts with this tag
7835		- $post_count = 0;
7836		- if ($term) {
7837		- $post_count = $term->count;
7838		- }
7839		-
7840		- $formatted_mappings[] = array(
7841		- 'tag_slug' => $tag_slug,
7842		- 'role_restriction' => $role_restriction,
7843		- 'role_label' => $role_options[$role_restriction] ?? $role_restriction,
7844		- 'post_count' => $post_count
7845		- );
7846		- }
7847		-
7848		- wp_send_json_success(array(
7849		- 'mappings' => $formatted_mappings
7850		- ));
7851		- exit;
7852		-}
7853		-
7854		-/**
7855		- * Bulk update role restrictions for all existing content with mapped tags
7856		- */
7857		-public function ajax_bulk_update_tag_roles() {
7858		- // Verify nonce and permissions
7859		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7860		-
7861		- if (!current_user_can('manage_options')) {
7862		- wp_send_json_error('Unauthorized access');
7863		- exit;
7864		- }
7865		-
7866		- // Get mappings
7867		- $mappings = get_option('mxchat_tag_role_mappings', array());
7868		-
7869		- if (empty($mappings)) {
7870		- wp_send_json_error('No tag-role mappings found');
7871		- exit;
7872		- }
7873		-
7874		- global $wpdb;
7875		-
7876		- // Check if using Pinecone
7877		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
7878		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7879		-
7880		- $updated_count = 0;
7881		- $details = array();
7882		-
7883		- foreach ($mappings as $tag_slug => $role_restriction) {
7884		- // Get all posts with this tag
7885		- $posts = get_posts(array(
7886		- 'tag' => $tag_slug,
7887		- 'post_type' => 'any',
7888		- 'posts_per_page' => -1,
7889		- 'fields' => 'ids',
7890		- 'post_status' => 'publish'
7891		- ));
7892		-
7893		- if (empty($posts)) {
7894		- continue;
7895		- }
7896		-
7897		- $tag_updated = 0;
7898		-
7899		- foreach ($posts as $post_id) {
7900		- $source_url = get_permalink($post_id);
7901		- if (!$source_url) {
7902		- continue;
7903		- }
7904		-
7905		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
7906		- // Update Pinecone role restriction
7907		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7908		- $vector_id = md5($source_url);
7909		-
7910		- $result = $wpdb->replace(
7911		- $roles_table,
7912		- array(
7913		- 'vector_id' => $vector_id,
7914		- 'role_restriction' => $role_restriction,
7915		- 'updated_at' => current_time('mysql')
7916		- ),
7917		- array('%s', '%s', '%s')
7918		- );
7919		- } else {
7920		- // Update WordPress DB
7921		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7922		-
7923		- $result = $wpdb->update(
7924		- $table_name,
7925		- array('role_restriction' => $role_restriction),
7926		- array('source_url' => $source_url),
7927		- array('%s'),
7928		- array('%s')
7929		- );
7930		- }
7931		-
7932		- if ($result !== false) {
7933		- $tag_updated++;
7934		- $updated_count++;
7935		- }
7936		- }
7937		-
7938		- if ($tag_updated > 0) {
7939		- $details[] = sprintf(
7940		- 'Tag "%s" (%s): %d posts updated',
7941		- $tag_slug,
7942		- $role_restriction,
7943		- $tag_updated
7944		- );
7945		- }
7946		- }
7947		-
7948		- wp_send_json_success(array(
7949		- 'message' => 'Bulk update completed',
7950		- 'updated_count' => $updated_count,
7951		- 'tags_processed' => count($mappings),
7952		- 'details' => $details
7953		- ));
7954		- exit;
7955		-}
7956		-
7957		-/**
7958		- * Handle tag changes on posts (when tags are added or removed)
7959		- */
7960		-public function handle_tag_change($object_id, $terms, $tt_ids, $taxonomy, $append, $old_tt_ids) {
7961		- // Only process post tags
7962		- if ($taxonomy !== 'post_tag') {
7963		- return;
7964		- }
7965		-
7966		- // Get tag-role mappings
7967		- $mappings = get_option('mxchat_tag_role_mappings', array());
7968		-
7969		- if (empty($mappings)) {
7970		- return;
7971		- }
7972		-
7973		- // Get the post's URL
7974		- $source_url = get_permalink($object_id);
7975		- if (!$source_url) {
7976		- return;
7977		- }
7978		-
7979		- // Determine the highest role restriction based on tags
7980		- $highest_role = 'public';
7981		- $role_hierarchy = array(
7982		- 'public' => 0,
7983		- 'logged_in' => 1,
7984		- 'subscriber' => 2,
7985		- 'contributor' => 3,
7986		- 'author' => 4,
7987		- 'editor' => 5,
7988		- 'administrator' => 6
7989		- );
7990		-
7991		- // Get all current tags for the post
7992		- $current_tags = wp_get_post_tags($object_id, array('fields' => 'slugs'));
7993		-
7994		- // Find the highest role restriction among the tags
7995		- foreach ($current_tags as $tag_slug) {
7996		- if (isset($mappings[$tag_slug])) {
7997		- $role = $mappings[$tag_slug];
7998		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
7999		- $highest_role = $role;
8000		- }
8001		- }
8002		- }
8003		-
8004		- // Update the role restriction in the database
8005		- global $wpdb;
8006		-
8007		- // Check if using Pinecone
8008		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
8009		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
8010		-
8011		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
8012		- // Update Pinecone role restriction
8013		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
8014		- $vector_id = md5($source_url);
8015		-
8016		- $wpdb->replace(
8017		- $roles_table,
8018		- array(
8019		- 'vector_id' => $vector_id,
8020		- 'role_restriction' => $highest_role,
8021		- 'updated_at' => current_time('mysql')
8022		- ),
8023		- array('%s', '%s', '%s')
8024		- );
8025		- } else {
8026		- // Update WordPress DB
8027		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
8028		-
8029		- $wpdb->update(
8030		- $table_name,
8031		- array('role_restriction' => $highest_role),
8032	3272	array('source_url' => $source_url),
8033		- array('%s'),
8034	3273	array('%s')
8035	3274	);
8036	3275	}
8037	3276	}
8038		-
8039		-/**
8040		- * Apply role restriction after content is stored (for auto-sync)
8041		- */
8042		-public function apply_role_restriction_after_storage($post_id, $source_url) {
8043		- // Get tag-role mappings
8044		- $mappings = get_option('mxchat_tag_role_mappings', array());
8045	3277
8046		- if (empty($mappings)) {
8047		- return;
8048		- }
8049		-
8050		- // Get all tags for the post
8051		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
8052		-
8053		- if (empty($post_tags)) {
8054		- return;
8055		- }
8056		-
8057		- // Determine the highest role restriction based on tags
8058		- $highest_role = 'public';
8059		- $role_hierarchy = array(
8060		- 'public' => 0,
8061		- 'logged_in' => 1,
8062		- 'subscriber' => 2,
8063		- 'contributor' => 3,
8064		- 'author' => 4,
8065		- 'editor' => 5,
8066		- 'administrator' => 6
8067		- );
8068		-
8069		- foreach ($post_tags as $tag_slug) {
8070		- if (isset($mappings[$tag_slug])) {
8071		- $role = $mappings[$tag_slug];
8072		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
8073		- $highest_role = $role;
8074		- }
8075		- }
8076		- }
8077		-
8078		- // If no restricted tags found, return (leave as public)
8079		- if ($highest_role === 'public') {
8080		- return;
8081		- }
8082		-
8083		- // Update the role restriction
8084		- global $wpdb;
8085		-
8086		- // Check if using Pinecone
8087		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
8088		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
8089		-
8090		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
8091		- // Update Pinecone role restriction
8092		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
8093		- $vector_id = md5($source_url);
8094		-
8095		- $wpdb->replace(
8096		- $roles_table,
8097		- array(
8098		- 'vector_id' => $vector_id,
8099		- 'role_restriction' => $highest_role,
8100		- 'updated_at' => current_time('mysql')
8101		- ),
8102		- array('%s', '%s', '%s')
8103		- );
8104		- } else {
8105		- // Update WordPress DB
8106		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
8107		-
8108		- $wpdb->update(
8109		- $table_name,
8110		- array('role_restriction' => $highest_role),
8111		- array('source_url' => $source_url),
8112		- array('%s'),
8113		- array('%s')
8114		- );
8115		- }
8116		-}
8117		-
8118		-
8119	3278	// ========================================
8120	3279	// HELPER METHODS
8121	3280	// ========================================
8122	3281
		@@ -8183,1039 +3342,8 @@
8183	3342	*/
8184	3343	private function mxchat_get_pinecone_manager() {
8185	3344	return MxChat_Pinecone_Manager::get_instance();
8186	3345	}
8187		-
8188		-
8189		- // ========================================
8190		-// DATABASE QUEUE TABLE MANAGEMENT
8191		-// ========================================
8192		-
8193		-/**
8194		- * Create queue table on plugin activation
8195		- * Call this from your plugin activation hook
8196		- */
8197		-public function mxchat_create_queue_table() {
8198		- global $wpdb;
8199		-
8200		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8201		- $charset_collate = $wpdb->get_charset_collate();
8202		-
8203		- $sql = "CREATE TABLE IF NOT EXISTS $table_name (
8204		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
8205		- queue_id varchar(64) NOT NULL,
8206		- item_type varchar(20) NOT NULL,
8207		- item_data longtext NOT NULL,
8208		- status varchar(20) NOT NULL DEFAULT 'pending',
8209		- bot_id varchar(50) NOT NULL DEFAULT 'default',
8210		- priority int(11) NOT NULL DEFAULT 0,
8211		- attempts int(11) NOT NULL DEFAULT 0,
8212		- max_attempts int(11) NOT NULL DEFAULT 3,
8213		- error_message text DEFAULT NULL,
8214		- created_at datetime NOT NULL,
8215		- started_at datetime DEFAULT NULL,
8216		- completed_at datetime DEFAULT NULL,
8217		- PRIMARY KEY (id),
8218		- KEY queue_id (queue_id),
8219		- KEY status (status),
8220		- KEY item_type (item_type),
8221		- KEY priority (priority)
8222		- ) $charset_collate;";
8223		-
8224		- require_once(ABSPATH . 'wp-admin/includes/upgrade.php');
8225		- dbDelta($sql);
8226		-
8227		- // Also create a meta table for queue metadata
8228		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
8229		-
8230		- $meta_sql = "CREATE TABLE IF NOT EXISTS $meta_table (
8231		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
8232		- queue_id varchar(64) NOT NULL,
8233		- meta_key varchar(255) NOT NULL,
8234		- meta_value longtext,
8235		- PRIMARY KEY (id),
8236		- KEY queue_id (queue_id),
8237		- KEY meta_key (meta_key)
8238		- ) $charset_collate;";
8239		-
8240		- dbDelta($meta_sql);
8241		-}
8242		-
8243		-/**
8244		- * Add items to the processing queue
8245		- *
8246		- * @param string $queue_id Unique identifier for this queue batch
8247		- * @param string $item_type Type of item (url, pdf_page)
8248		- * @param array $items Array of items to queue
8249		- * @param string $bot_id Bot ID for processing
8250		- * @return int Number of items queued
8251		- */
8252		-private function mxchat_add_to_queue($queue_id, $item_type, $items, $bot_id = 'default') {
8253		- global $wpdb;
8254		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8255		-
8256		- $queued_count = 0;
8257		- $priority = 0;
8258		-
8259		- foreach ($items as $item) {
8260		- $result = $wpdb->insert(
8261		- $table_name,
8262		- array(
8263		- 'queue_id' => $queue_id,
8264		- 'item_type' => $item_type,
8265		- 'item_data' => wp_json_encode($item),
8266		- 'status' => 'pending',
8267		- 'bot_id' => $bot_id,
8268		- 'priority' => $priority,
8269		- 'attempts' => 0,
8270		- 'max_attempts' => 3,
8271		- 'created_at' => current_time('mysql')
8272		- ),
8273		- array('%s', '%s', '%s', '%s', '%s', '%d', '%d', '%d', '%s')
8274		- );
8275		-
8276		- if ($result) {
8277		- $queued_count++;
8278		- }
8279		-
8280		- $priority++; // Process in order
8281		- }
8282		-
8283		- return $queued_count;
8284		-}
8285		-
8286		-/**
8287		- * Store queue metadata (total counts, source URL, etc.)
8288		- */
8289		-private function mxchat_set_queue_meta($queue_id, $meta_key, $meta_value) {
8290		- global $wpdb;
8291		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
8292		-
8293		- // Check if meta exists
8294		- $existing = $wpdb->get_var($wpdb->prepare(
8295		- "SELECT id FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
8296		- $queue_id,
8297		- $meta_key
8298		- ));
8299		-
8300		- if ($existing) {
8301		- // Update
8302		- $wpdb->update(
8303		- $meta_table,
8304		- array('meta_value' => maybe_serialize($meta_value)),
8305		- array('queue_id' => $queue_id, 'meta_key' => $meta_key),
8306		- array('%s'),
8307		- array('%s', '%s')
8308		- );
8309		- } else {
8310		- // Insert
8311		- $wpdb->insert(
8312		- $meta_table,
8313		- array(
8314		- 'queue_id' => $queue_id,
8315		- 'meta_key' => $meta_key,
8316		- 'meta_value' => maybe_serialize($meta_value)
8317		- ),
8318		- array('%s', '%s', '%s')
8319		- );
8320		- }
8321		-}
8322		-
8323		-/**
8324		- * Get queue metadata
8325		- */
8326		-private function mxchat_get_queue_meta($queue_id, $meta_key) {
8327		- global $wpdb;
8328		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
8329		-
8330		- $value = $wpdb->get_var($wpdb->prepare(
8331		- "SELECT meta_value FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
8332		- $queue_id,
8333		- $meta_key
8334		- ));
8335		-
8336		- return maybe_unserialize($value);
8337		-}
8338		-
8339		-// ========================================
8340		-// AJAX QUEUE PROCESSING HANDLERS
8341		-// ========================================
8342		-
8343		-/**
8344		- * AJAX: Get next item from queue to process
8345		- */
8346		-public function ajax_mxchat_get_next_queue_item() {
8347		- // Verify nonce and permissions
8348		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8349		-
8350		- if (!current_user_can('manage_options')) {
8351		- wp_send_json_error('Unauthorized access');
8352		- }
8353		-
8354		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8355		-
8356		- if (empty($queue_id)) {
8357		- wp_send_json_error('Missing queue ID');
8358		- }
8359		-
8360		- global $wpdb;
8361		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8362		-
8363		- // Get next pending item with retry logic for failed items
8364		- $next_item = $wpdb->get_row($wpdb->prepare(
8365		- "SELECT * FROM $table_name
8366		- WHERE queue_id = %s
8367		- AND status IN ('pending', 'failed')
8368		- AND attempts < max_attempts
8369		- ORDER BY priority ASC, id ASC
8370		- LIMIT 1",
8371		- $queue_id
8372		- ));
8373		-
8374		- if (!$next_item) {
8375		- // No more items - queue complete
8376		- wp_send_json_success(array(
8377		- 'complete' => true,
8378		- 'message' => 'Queue processing complete'
8379		- ));
8380		- }
8381		-
8382		- // Mark item as processing
8383		- $wpdb->update(
8384		- $table_name,
8385		- array(
8386		- 'status' => 'processing',
8387		- 'started_at' => current_time('mysql'),
8388		- 'attempts' => $next_item->attempts + 1
8389		- ),
8390		- array('id' => $next_item->id),
8391		- array('%s', '%s', '%d'),
8392		- array('%d')
8393		- );
8394		-
8395		- wp_send_json_success(array(
8396		- 'complete' => false,
8397		- 'item' => array(
8398		- 'id' => $next_item->id,
8399		- 'type' => $next_item->item_type,
8400		- 'data' => json_decode($next_item->item_data, true),
8401		- 'bot_id' => $next_item->bot_id,
8402		- 'attempt' => $next_item->attempts + 1
8403		- )
8404		- ));
8405		-}
8406		-
8407		-/**
8408		- * AJAX: Process a single queue item
8409		- */
8410		-public function ajax_mxchat_process_queue_item() {
8411		- // Verify nonce and permissions
8412		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8413		-
8414		- if (!current_user_can('manage_options')) {
8415		- wp_send_json_error('Unauthorized access');
8416		- }
8417		-
8418		- $item_id = isset($_POST['item_id']) ? absint($_POST['item_id']) : 0;
8419		- $item_type = isset($_POST['item_type']) ? sanitize_text_field($_POST['item_type']) : '';
8420		- $item_data = isset($_POST['item_data']) ? $_POST['item_data'] : array();
8421		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
8422		-
8423		- if (empty($item_id) || empty($item_type)) {
8424		- wp_send_json_error('Missing item data');
8425		- }
8426		-
8427		- global $wpdb;
8428		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8429		-
8430		- // Process based on item type
8431		- try {
8432		- set_time_limit(60); // Give processing 60 seconds
8433		-
8434		- $result = false;
8435		- $error_message = '';
8436		-
8437		- // Read item directly from DB to get queue_id and preserve special chars in item_data
8438		- // (POST round-trip through JS mangles characters like apostrophes in URLs)
8439		- $db_item = $wpdb->get_row($wpdb->prepare(
8440		- "SELECT queue_id, item_data FROM $table_name WHERE id = %d",
8441		- $item_id
8442		- ));
8443		- $item_queue_id = $db_item ? $db_item->queue_id : '';
8444		- if ($db_item && !empty($db_item->item_data)) {
8445		- $db_data = json_decode($db_item->item_data, true);
8446		- if (is_array($db_data)) {
8447		- $item_data = $db_data;
8448		- }
8449		- }
8450		-
8451		- switch ($item_type) {
8452		- case 'url':
8453		- $result = $this->mxchat_process_queue_url($item_data, $bot_id, $item_queue_id);
8454		- break;
8455		-
8456		- case 'pdf_page':
8457		- $result = $this->mxchat_process_queue_pdf_page($item_data, $bot_id);
8458		- break;
8459		-
8460		- default:
8461		- throw new Exception('Unknown item type: ' . $item_type);
8462		- }
8463		-
8464		- if (is_wp_error($result)) {
8465		- $error_code = $result->get_error_code();
8466		- // Content errors (empty page, sanitization) are permanent — retrying won't help
8467		- $permanent_codes = array('empty_page', 'empty_after_sanitization', 'no_api_key', 'page_not_found');
8468		- if (in_array($error_code, $permanent_codes)) {
8469		- // Mark as permanently failed — set attempts = max_attempts so it won't be retried
8470		- $current_item = $wpdb->get_row($wpdb->prepare(
8471		- "SELECT max_attempts FROM $table_name WHERE id = %d", $item_id
8472		- ));
8473		- $wpdb->update(
8474		- $table_name,
8475		- array(
8476		- 'status' => 'failed',
8477		- 'error_message' => $result->get_error_message(),
8478		- 'attempts' => $current_item ? $current_item->max_attempts : 3
8479		- ),
8480		- array('id' => $item_id),
8481		- array('%s', '%s', '%d'),
8482		- array('%d')
8483		- );
8484		- wp_send_json_error(array(
8485		- 'message' => $result->get_error_message(),
8486		- 'permanent_failure' => true,
8487		- 'item_id' => $item_id
8488		- ));
8489		- return;
8490		- }
8491		- throw new Exception($result->get_error_message());
8492		- }
8493		-
8494		- if ($result === false) {
8495		- throw new Exception('Processing returned false - item may be empty or invalid');
8496		- }
8497		-
8498		- // Mark as completed
8499		- $wpdb->update(
8500		- $table_name,
8501		- array(
8502		- 'status' => 'completed',
8503		- 'completed_at' => current_time('mysql'),
8504		- 'error_message' => null
8505		- ),
8506		- array('id' => $item_id),
8507		- array('%s', '%s', '%s'),
8508		- array('%d')
8509		- );
8510		-
8511		- wp_send_json_success(array(
8512		- 'processed' => true,
8513		- 'item_id' => $item_id,
8514		- 'message' => 'Item processed successfully'
8515		- ));
8516		-
8517		- } catch (Exception $e) {
8518		- $error_message = $e->getMessage();
8519		-
8520		- // Get current attempt count
8521		- $item = $wpdb->get_row($wpdb->prepare(
8522		- "SELECT attempts, max_attempts FROM $table_name WHERE id = %d",
8523		- $item_id
8524		- ));
8525		-
8526		- // Check if we've exhausted retries
8527		- if ($item && $item->attempts >= $item->max_attempts) {
8528		- // Permanently failed
8529		- $wpdb->update(
8530		- $table_name,
8531		- array(
8532		- 'status' => 'failed',
8533		- 'error_message' => $error_message
8534		- ),
8535		- array('id' => $item_id),
8536		- array('%s', '%s'),
8537		- array('%d')
8538		- );
8539		-
8540		- wp_send_json_error(array(
8541		- 'message' => 'Item failed after maximum attempts: ' . $error_message,
8542		- 'permanent_failure' => true,
8543		- 'item_id' => $item_id
8544		- ));
8545		- } else {
8546		- // Mark for retry
8547		- $wpdb->update(
8548		- $table_name,
8549		- array(
8550		- 'status' => 'failed',
8551		- 'error_message' => $error_message
8552		- ),
8553		- array('id' => $item_id),
8554		- array('%s', '%s'),
8555		- array('%d')
8556		- );
8557		-
8558		- wp_send_json_error(array(
8559		- 'message' => 'Item processing failed, will retry: ' . $error_message,
8560		- 'can_retry' => true,
8561		- 'item_id' => $item_id,
8562		- 'attempts' => $item ? $item->attempts : 0
8563		- ));
8564		- }
8565		- }
8566		-}
8567		-
8568		-/**
8569		- * Process a URL from the queue
8570		- */
8571		-private function mxchat_process_queue_url($item_data, $bot_id = 'default', $queue_id = '') {
8572		- $url = isset($item_data['url']) ? $item_data['url'] : '';
8573		-
8574		- if (empty($url)) {
8575		- return new WP_Error('invalid_url', 'URL is empty');
8576		- }
8577		-
8578		- // Get bot-specific embedding decision early (needed for both paths) —
8579		- // custom-provider-aware (plan cbd5fd). Error code preserved.
8580		- $bot_options = $this->get_bot_options($bot_id);
8581		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
8582		-
8583		- $preflight = MxChat_Utils::embedding_preflight($options);
8584		- if (!$preflight['ok']) {
8585		- return new WP_Error('no_api_key', $preflight['reason'] . ' (bot: ' . $bot_id . ')');
8586		- }
8587		- $api_key = $preflight['api_key'];
8588		-
8589		- // Check if this is a WooCommerce product URL and WooCommerce is active
8590		- $is_product_url = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
8591		- $content_type = $is_product_url ? 'product' : 'url';
8592		-
8593		- // Try to get WooCommerce product data if it's a product URL
8594		- if ($is_product_url && class_exists('WooCommerce')) {
8595		- $product_content = $this->mxchat_extract_woocommerce_product_content($url);
8596		-
8597		- if (!empty($product_content)) {
8598		- // Successfully extracted WooCommerce product data with pricing
8599		- $result = MxChat_Utils::submit_content_to_db(
8600		- $product_content,
8601		- $url,
8602		- $api_key,
8603		- null,
8604		- $bot_id,
8605		- 'product'
8606		- );
8607		- return $result;
8608		- }
8609		- // If WooCommerce extraction failed, fall through to HTML extraction
8610		- }
8611		-
8612		- // Fetch URL content (fallback for non-products or when WooCommerce extraction fails)
8613		- $is_likely_pdf = (strtolower(pathinfo(parse_url($url, PHP_URL_PATH) ?: '', PATHINFO_EXTENSION)) === 'pdf');
8614		- $response = wp_remote_get($url, array(
8615		- 'timeout' => $is_likely_pdf ? 120 : 30,
8616		- 'redirection' => 5,
8617		- 'user-agent' => mxchat_ingest_user_agent(),
8618		- ));
8619		-
8620		- if (is_wp_error($response)) {
8621		- return $response;
8622		- }
8623		-
8624		- $response_code = wp_remote_retrieve_response_code($response);
8625		- if ($response_code !== 200) {
8626		- return new WP_Error('http_error', 'HTTP ' . $response_code . ' error for: ' . $url);
8627		- }
8628		-
8629		- // Check if URL is a PDF — expand into per-page queue items using the standard PDF pipeline
8630		- if ($this->mxchat_is_pdf_url($url, $response)) {
8631		- return $this->mxchat_expand_pdf_to_queue($url, $response, $bot_id, $queue_id);
8632		- }
8633		-
8634		- $html = wp_remote_retrieve_body($response);
8635		-
8636		- if (empty($html)) {
8637		- return new WP_Error('empty_response', 'Empty response body');
8638		- }
8639		-
8640		- // Extract and sanitize content
8641		- $content = $this->mxchat_extract_main_content($html);
8642		- $sanitized = $this->mxchat_sanitize_content_for_api($content);
8643		-
8644		- if (empty($sanitized)) {
8645		- // Not an error - just no content found (maybe a redirect or empty page)
8646		- return false;
8647		- }
8648		-
8649		- // Submit to database with content_type
8650		- $result = MxChat_Utils::submit_content_to_db(
8651		- $sanitized,
8652		- $url,
8653		- $api_key,
8654		- null,
8655		- $bot_id,
8656		- $content_type
8657		- );
8658		-
8659		- return $result;
8660		-}
8661		-
8662		-/**
8663		- * Expand a PDF URL into per-page queue items using the standard PDF pipeline.
8664		- * Called when a sitemap URL turns out to be a PDF — downloads, parses page count,
8665		- * and adds pdf_page items to the same queue so they process with full progress tracking.
8666		- */
8667		-private function mxchat_expand_pdf_to_queue($pdf_url, $response, $bot_id = 'default', $queue_id = '') {
8668		- set_time_limit(120); // PDFs need extra time for download + parsing
8669		-
8670		- $upload_dir = wp_upload_dir();
8671		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
8672		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
8673		-
8674		- $response_body = wp_remote_retrieve_body($response);
8675		- if (empty($response_body)) {
8676		- return new WP_Error('empty_pdf', 'Empty PDF response for: ' . $pdf_url);
8677		- }
8678		-
8679		- if (!wp_mkdir_p(dirname($pdf_path))) {
8680		- return new WP_Error('dir_error', 'Failed to create upload directory');
8681		- }
8682		-
8683		- file_put_contents($pdf_path, $response_body);
8684		-
8685		- if (!file_exists($pdf_path)) {
8686		- return new WP_Error('save_error', 'Failed to save PDF file');
8687		- }
8688		-
8689		- try {
8690		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
8691		-
8692		- if ($total_pages === false || $total_pages < 1) {
8693		- wp_delete_file($pdf_path);
8694		- return new WP_Error('no_pages', 'PDF has no pages: ' . $pdf_url);
8695		- }
8696		-
8697		- // Build per-page items identical to mxchat_handle_pdf_for_knowledge_base
8698		- $pages = array();
8699		- for ($i = 1; $i <= $total_pages; $i++) {
8700		- $pages[] = array(
8701		- 'pdf_path' => $pdf_path,
8702		- 'pdf_url' => $pdf_url,
8703		- 'page_number' => $i,
8704		- 'total_pages' => $total_pages
8705		- );
8706		- }
8707		-
8708		- // Add pdf_page items to the SAME queue so the JS picks them up automatically
8709		- if (!empty($queue_id)) {
8710		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
8711		- } else {
8712		- // Fallback: create a new PDF queue (shouldn't happen in sitemap flow)
8713		- $new_queue_id = 'pdf_' . md5($pdf_url . time());
8714		- $queued_count = $this->mxchat_add_to_queue($new_queue_id, 'pdf_page', $pages, $bot_id);
8715		- $this->mxchat_set_queue_meta($new_queue_id, 'source_url', $pdf_url);
8716		- $this->mxchat_set_queue_meta($new_queue_id, 'queue_type', 'pdf');
8717		- $this->mxchat_set_queue_meta($new_queue_id, 'total_items', $total_pages);
8718		- $this->mxchat_set_queue_meta($new_queue_id, 'bot_id', $bot_id);
8719		- $this->mxchat_set_queue_meta($new_queue_id, 'pdf_path', $pdf_path);
8720		- $this->mxchat_set_queue_meta($new_queue_id, 'created_at', current_time('mysql'));
8721		- }
8722		-
8723		- if ($queued_count === 0) {
8724		- wp_delete_file($pdf_path);
8725		- return new WP_Error('queue_error', 'Failed to add PDF pages to queue');
8726		- }
8727		-
8728		- // Return true so the original URL item is marked complete
8729		- // The new pdf_page items will be processed in subsequent batches
8730		- return true;
8731		-
8732		- } catch (Exception $e) {
8733		- if (file_exists($pdf_path)) {
8734		- wp_delete_file($pdf_path);
8735		- }
8736		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
8737		- }
8738		-}
8739		-
8740		-/**
8741		- * Legacy: Process a PDF URL inline during sitemap queue processing.
8742		- * @deprecated Use mxchat_expand_pdf_to_queue instead — kept for reference only.
8743		- */
8744		-private function mxchat_process_pdf_url_inline($pdf_url, $response, $api_key, $bot_id = 'default') {
8745		- set_time_limit(120); // PDFs need more time — downloading + parsing all pages
8746		-
8747		- $upload_dir = wp_upload_dir();
8748		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
8749		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
8750		-
8751		- $response_body = wp_remote_retrieve_body($response);
8752		- if (empty($response_body)) {
8753		- return new WP_Error('empty_pdf', 'Empty PDF response');
8754		- }
8755		-
8756		- if (!wp_mkdir_p(dirname($pdf_path))) {
8757		- return new WP_Error('dir_error', 'Failed to create upload directory');
8758		- }
8759		-
8760		- file_put_contents($pdf_path, $response_body);
8761		-
8762		- if (!file_exists($pdf_path)) {
8763		- return new WP_Error('save_error', 'Failed to save PDF file');
8764		- }
8765		-
8766		- try {
8767		- mxchat_load_pdf_parser();
8768		- $parser = new \Smalot\PdfParser\Parser();
8769		- $pdf = $parser->parseFile($pdf_path);
8770		- $pages = $pdf->getPages();
8771		- $total_pages = count($pages);
8772		-
8773		- if ($total_pages < 1) {
8774		- wp_delete_file($pdf_path);
8775		- return new WP_Error('no_pages', 'PDF has no pages');
8776		- }
8777		-
8778		- $processed = 0;
8779		- $skipped_pages = array();
8780		-
8781		- for ($i = 0; $i < $total_pages; $i++) {
8782		- $page_num = $i + 1;
8783		- $text = $pages[$i]->getText();
8784		- $text = MxChat_Utils::normalize_pdf_rtl($text, 'kb_pdf_import page ' . $page_num);
8785		- if (empty($text)) {
8786		- $skipped_pages[] = 'Page ' . $page_num . ': No text could be extracted — page may contain only images, links, or non-standard encoding';
8787		- continue;
8788		- }
8789		-
8790		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
8791		- if (empty($sanitized)) {
8792		- $skipped_pages[] = 'Page ' . $page_num . ': Text was extracted but contained only special characters, control codes, or unsupported content';
8793		- continue;
8794		- }
8795		-
8796		- $metadata = array(
8797		- 'document_type' => 'pdf',
8798		- 'total_pages' => $total_pages,
8799		- 'current_page' => $page_num,
8800		- 'source_url' => $pdf_url,
8801		- );
8802		-
8803		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
8804		- $page_url = esc_url($pdf_url . '#page=' . $page_num);
8805		-
8806		- MxChat_Utils::submit_content_to_db(
8807		- $content_with_metadata,
8808		- $page_url,
8809		- $api_key,
8810		- null,
8811		- $bot_id,
8812		- 'pdf'
8813		- );
8814		-
8815		- $processed++;
8816		- }
8817		-
8818		- // Clean up the temp PDF file
8819		- wp_delete_file($pdf_path);
8820		-
8821		- if (!empty($skipped_pages)) {
8822		- error_log('MxChat PDF: Skipped ' . count($skipped_pages) . ' of ' . $total_pages . ' pages: ' . implode('; ', $skipped_pages));
8823		- }
8824		-
8825		- return $processed > 0 ? true : false;
8826		-
8827		- } catch (Exception $e) {
8828		- if (file_exists($pdf_path)) {
8829		- wp_delete_file($pdf_path);
8830		- }
8831		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
8832		- }
8833		-}
8834		-
8835		-/**
8836		- * Extract WooCommerce product content including pricing
8837		- *
8838		- * @param string $url The product URL
8839		- * @return string|false Product content with pricing, or false if not found
8840		- */
8841		-private function mxchat_extract_woocommerce_product_content($url) {
8842		- // Try to get product ID from URL
8843		- $product_id = url_to_postid($url);
8844		-
8845		- // If url_to_postid fails, try to extract from URL pattern
8846		- if (!$product_id) {
8847		- $product_slug = '';
8848		-
8849		- // Handle pretty permalinks: /product/product-name/
8850		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
8851		- $product_slug = $matches[1];
8852		- }
8853		-
8854		- if (!empty($product_slug)) {
8855		- $product_post = get_page_by_path($product_slug, OBJECT, 'product');
8856		- if ($product_post) {
8857		- $product_id = $product_post->ID;
8858		- }
8859		- }
8860		- }
8861		-
8862		- if (!$product_id) {
8863		- return false;
8864		- }
8865		-
8866		- // Get WooCommerce product object
8867		- $product = wc_get_product($product_id);
8868		-
8869		- if (!$product) {
8870		- return false;
8871		- }
8872		-
8873		- // Build product content with pricing (similar to mxchat_store_product_embedding)
8874		- $title = $product->get_name();
8875		- $description = $product->get_description();
8876		- $short_description = $product->get_short_description();
8877		- $sku = $product->get_sku();
8878		-
8879		- // Format content
8880		- $content = $title . "\n\n";
8881		-
8882		- if (!empty($short_description)) {
8883		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
8884		- }
8885		-
8886		- if (!empty($description)) {
8887		- $content .= wp_strip_all_tags($description) . "\n\n";
8888		- }
8889		-
8890		- // Add pricing information (base-currency pinned — see mxchat_product_price_lines)
8891		- $content .= $this->mxchat_product_price_lines($product);
8892		-
8893		- if (!empty($sku)) {
8894		- $content .= "SKU: " . $sku . "\n";
8895		- }
8896		-
8897		- // Get product categories
8898		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
8899		- if (!empty($categories) && !is_wp_error($categories)) {
8900		- $content .= "Categories: " . implode(', ', $categories) . "\n";
8901		- }
8902		-
8903		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
8904		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
8905		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
8906		- foreach ($custom_tabs as $tab) {
8907		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
8908		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
8909		-
8910		- if (!empty($tab_title) && !empty($tab_content)) {
8911		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
8912		- }
8913		- }
8914		- }
8915		-
8916		- // Also check for reusable/saved tabs applied to this product
8917		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
8918		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
8919		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
8920		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
8921		- foreach ($applied_saved_tabs as $saved_tab_id) {
8922		- if (isset($saved_tabs[$saved_tab_id])) {
8923		- $tab = $saved_tabs[$saved_tab_id];
8924		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
8925		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
8926		-
8927		- if (!empty($tab_title) && !empty($tab_content)) {
8928		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
8929		- }
8930		- }
8931		- }
8932		- }
8933		- }
8934		-
8935		- return $this->mxchat_sanitize_content_for_api($content);
8936		-}
8937		-
8938		-/**
8939		- * Process a PDF page from the queue
8940		- */
8941		-private function mxchat_process_queue_pdf_page($item_data, $bot_id = 'default') {
8942		- $pdf_path = isset($item_data['pdf_path']) ? $item_data['pdf_path'] : '';
8943		- $pdf_url = isset($item_data['pdf_url']) ? $item_data['pdf_url'] : '';
8944		- $page_number = isset($item_data['page_number']) ? absint($item_data['page_number']) : 0;
8945		- $total_pages = isset($item_data['total_pages']) ? absint($item_data['total_pages']) : 0;
8946		-
8947		- if (empty($pdf_path) || !file_exists($pdf_path)) {
8948		- return new WP_Error('pdf_not_found', 'PDF file not found: ' . $pdf_path);
8949		- }
8950		-
8951		- if ($page_number < 1) {
8952		- return new WP_Error('invalid_page', 'Invalid page number');
8953		- }
8954		-
8955		- try {
8956		- mxchat_load_pdf_parser();
8957		- $parser = new \Smalot\PdfParser\Parser();
8958		- $pdf = $parser->parseFile($pdf_path);
8959		- $pages = $pdf->getPages();
8960		-
8961		- if (!isset($pages[$page_number - 1])) {
8962		- return new WP_Error('page_not_found', 'Page ' . $page_number . ' not found in PDF');
8963		- }
8964		-
8965		- $text = $pages[$page_number - 1]->getText();
8966		- $text = MxChat_Utils::normalize_pdf_rtl($text, 'kb_pdf_page page ' . $page_number);
8967		-
8968		- if (empty($text)) {
8969		- return new WP_Error('empty_page', 'Page ' . $page_number . ': No text could be extracted — page may contain only images, links, or non-standard encoding');
8970		- }
8971		-
8972		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
8973		-
8974		- if (empty($sanitized)) {
8975		- return new WP_Error('empty_after_sanitization', 'Page ' . $page_number . ': Text was extracted but contained only special characters, control codes, or unsupported content that was removed during cleanup');
8976		- }
8977		-
8978		- // Create metadata
8979		- $metadata = array(
8980		- 'document_type' => 'pdf',
8981		- 'total_pages' => $total_pages,
8982		- 'current_page' => $page_number,
8983		- 'source_url' => $pdf_url
8984		- );
8985		-
8986		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
8987		- $page_url = esc_url($pdf_url . "#page=" . $page_number);
8988		-
8989		- // Get bot-specific embedding decision — custom-provider-aware
8990		- // (plan cbd5fd). Error code preserved.
8991		- $bot_options = $this->get_bot_options($bot_id);
8992		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
8993		-
8994		- $preflight = MxChat_Utils::embedding_preflight($options);
8995		- if (!$preflight['ok']) {
8996		- return new WP_Error('no_api_key', $preflight['reason'] . ' (bot: ' . $bot_id . ')');
8997		- }
8998		- $api_key = $preflight['api_key'];
8999		-
9000		- // Submit to database - UPDATED 2.5.6: Added content_type 'pdf'
9001		- $result = MxChat_Utils::submit_content_to_db(
9002		- $content_with_metadata,
9003		- $page_url,
9004		- $api_key,
9005		- null,
9006		- $bot_id,
9007		- 'pdf'
9008		- );
9009		-
9010		- return $result;
9011		-
9012		- } catch (Exception $e) {
9013		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
9014		- }
9015		-}
9016		-
9017		-/**
9018		- * AJAX: Get queue processing status
9019		- */
9020		-public function ajax_mxchat_get_queue_status() {
9021		- // Verify nonce and permissions
9022		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
9023		-
9024		- if (!current_user_can('manage_options')) {
9025		- wp_send_json_error('Unauthorized access');
9026		- }
9027		-
9028		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
9029		-
9030		- if (empty($queue_id)) {
9031		- wp_send_json_error('Missing queue ID');
9032		- }
9033		-
9034		- global $wpdb;
9035		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
9036		-
9037		- // Get counts by status
9038		- $counts = $wpdb->get_results($wpdb->prepare(
9039		- "SELECT status, COUNT(*) as count
9040		- FROM $table_name
9041		- WHERE queue_id = %s
9042		- GROUP BY status",
9043		- $queue_id
9044		- ), OBJECT_K);
9045		-
9046		- $total = 0;
9047		- $completed = 0;
9048		- $failed = 0;
9049		- $processing = 0;
9050		- $pending = 0;
9051		-
9052		- foreach ($counts as $status => $data) {
9053		- $count = absint($data->count);
9054		- $total += $count;
9055		-
9056		- switch ($status) {
9057		- case 'completed':
9058		- $completed = $count;
9059		- break;
9060		- case 'failed':
9061		- $failed = $count;
9062		- break;
9063		- case 'processing':
9064		- $processing = $count;
9065		- break;
9066		- case 'pending':
9067		- $pending = $count;
9068		- break;
9069		- }
9070		- }
9071		-
9072		- // Calculate percentage
9073		- $percentage = $total > 0 ? round((($completed + $failed) / $total) * 100) : 0;
9074		-
9075		- // Get failed items details (include all failed items, not just those that exhausted retries)
9076		- $failed_items = array();
9077		- if ($failed > 0) {
9078		- $failed_items = $wpdb->get_results($wpdb->prepare(
9079		- "SELECT item_type, item_data, error_message, attempts
9080		- FROM $table_name
9081		- WHERE queue_id = %s
9082		- AND status = 'failed'
9083		- ORDER BY id DESC
9084		- LIMIT 50",
9085		- $queue_id
9086		- ));
9087		- }
9088		-
9089		- // Get queue metadata
9090		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
9091		- $queue_type = $this->mxchat_get_queue_meta($queue_id, 'queue_type');
9092		-
9093		- // Determine if queue is complete
9094		- $is_complete = ($pending === 0 && $processing === 0);
9095		-
9096		- wp_send_json_success(array(
9097		- 'queue_id' => $queue_id,
9098		- 'queue_type' => $queue_type,
9099		- 'source_url' => $source_url,
9100		- 'total' => $total,
9101		- 'completed' => $completed,
9102		- 'failed' => $failed,
9103		- 'processing' => $processing,
9104		- 'pending' => $pending,
9105		- 'percentage' => $percentage,
9106		- 'is_complete' => $is_complete,
9107		- 'failed_items' => $failed_items,
9108		- 'status' => $is_complete ? 'complete' : 'processing'
9109		- ));
9110		-}
9111		-
9112		-/**
9113		- * AJAX: Clear completed queue
9114		- */
9115		-public function ajax_mxchat_clear_queue() {
9116		- // Verify nonce and permissions
9117		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
9118		-
9119		- if (!current_user_can('manage_options')) {
9120		- wp_send_json_error('Unauthorized access');
9121		- }
9122		-
9123		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
9124		-
9125		- if (empty($queue_id)) {
9126		- wp_send_json_error('Missing queue ID');
9127		- }
9128		-
9129		- global $wpdb;
9130		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
9131		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
9132		-
9133		- // Delete queue items
9134		- $wpdb->delete(
9135		- $table_name,
9136		- array('queue_id' => $queue_id),
9137		- array('%s')
9138		- );
9139		-
9140		- // Delete queue metadata
9141		- $wpdb->delete(
9142		- $meta_table,
9143		- array('queue_id' => $queue_id),
9144		- array('%s')
9145		- );
9146		-
9147		- wp_send_json_success(array(
9148		- 'message' => 'Queue cleared successfully'
9149		- ));
9150		-}
9151		-
9152		-/**
9153		- * AJAX: Retry failed items in queue
9154		- */
9155		-public function ajax_mxchat_retry_failed() {
9156		- // Verify nonce and permissions
9157		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
9158		-
9159		- if (!current_user_can('manage_options')) {
9160		- wp_send_json_error('Unauthorized access');
9161		- }
9162		-
9163		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
9164		-
9165		- if (empty($queue_id)) {
9166		- wp_send_json_error('Missing queue ID');
9167		- }
9168		-
9169		- global $wpdb;
9170		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
9171		-
9172		- // Reset failed items to pending and reset attempt count
9173		- $updated = $wpdb->update(
9174		- $table_name,
9175		- array(
9176		- 'status' => 'pending',
9177		- 'attempts' => 0,
9178		- 'error_message' => null
9179		- ),
9180		- array(
9181		- 'queue_id' => $queue_id,
9182		- 'status' => 'failed'
9183		- ),
9184		- array('%s', '%d', '%s'),
9185		- array('%s', '%s')
9186		- );
9187		-
9188		- wp_send_json_success(array(
9189		- 'message' => 'Reset ' . $updated . ' failed items for retry',
9190		- 'reset_count' => $updated
9191		- ));
9192		-}
9193		-
9194		-
9195		-public function ajax_mxchat_mark_queue_complete() {
9196		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
9197		-
9198		- if (!current_user_can('manage_options')) {
9199		- wp_send_json_error('Unauthorized access');
9200		- }
9201		-
9202		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
9203		-
9204		- if (empty($queue_id)) {
9205		- wp_send_json_error('Missing queue ID');
9206		- }
9207		-
9208		- // Clear active queue transients
9209		- if (strpos($queue_id, 'sitemap_') === 0) {
9210		- delete_transient('mxchat_active_queue_sitemap');
9211		- } else if (strpos($queue_id, 'pdf_') === 0) {
9212		- delete_transient('mxchat_active_queue_pdf');
9213		- }
9214		-
9215		- wp_send_json_success(array('message' => 'Queue marked as complete'));
9216		-}
9217		-
9218	3346
9219	3347	// ========================================
9220	3348	// STATIC ACCESS METHODS
9221	3349	// ========================================