← All changes | admin/class-knowledge-manager.php +1702 -6971 3.2.17 → 2.2.7 View file →

		@@ -9,101 +9,57 @@
9	9	exit; // Exit if accessed directly
10	10	}
11	11
12	12	class MxChat_Knowledge_Manager {
13		-
	13	+
14	14	private $options;
15		-
16		- // Post IDs whose vectors were already deleted by mxchat_handle_status_transition this
17		- // request, so the transient-based branch in mxchat_handle_post_update can skip the
18		- // redundant (idempotent but network-visible) second deletion.
19		- private $transition_deleted_posts = array();
20		-
21		- // Post IDs already INDEXED by mxchat_handle_status_transition's arrival edge this
22		- // request. Normal editor publishes fire transition_post_status first, then
23		- // post_updated — without this guard every editor publish would embed twice.
24		- private $transition_indexed_posts = array();
25		-
	15	+
26	16	/**
27	17	* Constructor - Register hooks for content processing
28	18	*/
29		-public function __construct() {
30		- $this->options = get_option('mxchat_options', array());
31		- $this->mxchat_init_hooks();
	19	+ public function __construct() {
	20	+ $this->options = get_option('mxchat_options', array());
	21	+ $this->mxchat_init_hooks();
	22	+ }
32	23
33		- $this->mxchat_init_role_hooks();
34		-}
35		-
36		-/**
37		- * Initialize WordPress hooks for content processing
38		- *
39		- */
40		-private function mxchat_init_hooks() {
41		- // Admin post handlers for form submissions
42		- add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
43		- add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
44		- add_action('admin_post_mxchat_submit_pdf_file', array($this, 'mxchat_handle_pdf_file_submission'));
45		- add_action('admin_post_mxchat_submit_youtube', array($this, 'mxchat_handle_youtube_submission'));
46		- add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
47		-
48		- // AJAX handlers for real-time processing and status updates
49		- add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
50		- add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status'));
51		- add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
52		- add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
53		- add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
54		- add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
55		- add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
56		- add_action('wp_ajax_mxchat_delete_chunks_by_url', array($this, 'ajax_mxchat_delete_chunks_by_url'));
57		- add_action('wp_ajax_mxchat_delete_wordpress_prompt', array($this, 'ajax_mxchat_delete_wordpress_prompt'));
58		- add_action('wp_ajax_mxchat_bulk_delete_knowledge', array($this, 'ajax_mxchat_bulk_delete_knowledge'));
59		- add_action('wp_ajax_mxchat_update_role_restriction', array($this, 'ajax_mxchat_update_role_restriction'));
60		-
61		- // Queue-based processing AJAX handlers
62		- add_action('wp_ajax_mxchat_get_next_queue_item', array($this, 'ajax_mxchat_get_next_queue_item'));
63		- add_action('wp_ajax_mxchat_process_queue_item', array($this, 'ajax_mxchat_process_queue_item'));
64		- add_action('wp_ajax_mxchat_get_queue_status', array($this, 'ajax_mxchat_get_queue_status'));
65		- add_action('wp_ajax_mxchat_clear_queue', array($this, 'ajax_mxchat_clear_queue'));
66		- add_action('wp_ajax_mxchat_retry_failed', array($this, 'ajax_mxchat_retry_failed'));
67		- add_action('wp_ajax_mxchat_get_recent_entries', array($this, 'ajax_mxchat_get_recent_entries'));
68		- add_action('wp_ajax_mxchat_detect_sitemaps', array($this, 'ajax_mxchat_detect_sitemaps'));
69		- add_action('wp_ajax_mxchat_refresh_pinecone_entries', array($this, 'ajax_mxchat_refresh_pinecone_entries'));
70		- add_action('wp_ajax_mxchat_paginate_entries', array($this, 'ajax_mxchat_paginate_entries'));
71		- add_action('wp_ajax_mxchat_get_entry_content', array($this, 'ajax_mxchat_get_entry_content'));
72		- add_action('wp_ajax_mxchat_save_entry_content', array($this, 'ajax_mxchat_save_entry_content'));
73		- add_action('wp_ajax_mxchat_inspect_entry', array($this, 'ajax_mxchat_inspect_entry'));
	24	+ /**
	25	+ * Initialize WordPress hooks for content processing
	26	+ */
	27	+ private function mxchat_init_hooks() {
	28	+ // Admin post handlers for form submissions
	29	+ add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
	30	+ add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
	31	+ add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
	32	+
	33	+ // AJAX handlers for real-time processing and status updates
	34	+ add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
	35	+ add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status')); // NEW
	36	+ add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
	37	+ add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
	38	+ add_action('wp_ajax_mxchat_manual_batch_process', array($this, 'ajax_manual_batch_process'));
	39	+ add_action('mxchat_delete_content', array($this, 'mxchat_delete_from_pinecone_by_url'), 10, 1);
74	40
75		- // WordPress post management hooks
76		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
77		- add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
78		- add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
79		- add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
80		- // Authoritative unpublish detection: core hands this hook the REAL previous status, so
81		- // removal no longer depends on the mxchat_prev_status_* transients (evictable by persistent
82		- // object caches, never written by paths that bypass wp_update_post, e.g. plugins flipping
83		- // post_status directly and calling wp_transition_post_status themselves).
84		- add_action('transition_post_status', array($this, 'mxchat_handle_status_transition'), 10, 3);
85		-
86		- // ACF hook - fires AFTER ACF fields are saved, ensuring ACF data is available
87		- // Priority 20 to run after ACF's own save (which runs at priority 10)
88		- add_action('acf/save_post', array($this, 'mxchat_handle_acf_save'), 20);
89		-
90		- // One-time cleanup for vectors orphaned by unpublishes that predate the
91		- // transition_post_status handler (plan 816fb1): wp mxchat prune-unpublished
92		- if (defined('WP_CLI') && WP_CLI) {
93		- WP_CLI::add_command('mxchat prune-unpublished', array($this, 'cli_prune_unpublished'));
	41	+
	42	+ // Cron handlers for background processing
	43	+ add_action('mxchat_process_sitemap_urls', array($this, 'mxchat_process_sitemap_urls_cron'), 10, 5);
	44	+ add_action('mxchat_process_pdf_pages', array($this, 'mxchat_process_pdf_pages_cron'), 10, 5);
	45	+
	46	+ // WordPress post management hooks
	47	+ add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
	48	+ add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
	49	+ add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
	50	+ add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
	51	+ add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
	52	+ add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
	53	+
	54	+ // WooCommerce product hooks (if WooCommerce is active)
	55	+ if (class_exists('WooCommerce')) {
	56	+ add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
	57	+ add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
	58	+ add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
	59	+ }
	60	+
94	61	}
95		-
96		- add_action('wp_ajax_mxchat_mark_queue_complete', array($this, 'ajax_mxchat_mark_queue_complete'));
97		-
98		- // WooCommerce product hooks (if WooCommerce is active)
99		- if (class_exists('WooCommerce')) {
100		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
101		- add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
102		- add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
103		- add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
104		- }
105		-}
106	62
107	63	/**
108	64	* Get current options (refreshed)
109	65	*/
		@@ -113,9 +69,315 @@
113	69	}
114	70	return $this->options;
115	71	}
116	72
	73	+
	74	+ /**
	75	+ * Handle manual batch processing via AJAX
	76	+ */
	77	+public function ajax_manual_batch_process() {
	78	+ try {
	79	+ // Verify nonce and permissions
	80	+ check_ajax_referer('mxchat_status_nonce', 'nonce');
	81	+
	82	+ if (!current_user_can('manage_options')) {
	83	+ wp_send_json_error('Unauthorized access');
	84	+ }
	85	+
	86	+ $process_type = sanitize_text_field($_POST['process_type'] ?? '');
	87	+ $url = sanitize_text_field($_POST['url'] ?? '');
	88	+
	89	+ if (empty($process_type) || empty($url)) {
	90	+ wp_send_json_error('Missing required parameters');
	91	+ }
	92	+
	93	+ $processed = 0;
	94	+
	95	+ if ($process_type === 'pdf') {
	96	+ $processed = $this->mxchat_manual_process_pdf_batch($url);
	97	+ } elseif ($process_type === 'sitemap') {
	98	+ $processed = $this->mxchat_manual_process_sitemap_batch($url);
	99	+ }
	100	+
	101	+ if ($processed > 0) {
	102	+ wp_send_json_success(array(
	103	+ 'message' => "Processed {$processed} items successfully",
	104	+ 'processed' => $processed
	105	+ ));
	106	+ } else {
	107	+ wp_send_json_error('No items were processed');
	108	+ }
	109	+
	110	+ } catch (Exception $e) {
	111	+ //error_log('Manual batch process error: ' . $e->getMessage());
	112	+ wp_send_json_error('Processing failed: ' . $e->getMessage());
	113	+ }
	114	+}
117	115
	116	+/**
	117	+ * Process a small PDF batch manually - DIRECT PROCESSING
	118	+ */
	119	+private function mxchat_manual_process_pdf_batch($pdf_url) {
	120	+ try {
	121	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	122	+ $status = get_transient($status_key);
	123	+
	124	+ if (!$status || $status['status'] !== 'processing') {
	125	+ //error_log('Manual PDF: No processing status found');
	126	+ return 0;
	127	+ }
	128	+
	129	+ //error_log('Manual PDF: Starting direct processing for ' . $pdf_url);
	130	+
	131	+ // Get current progress
	132	+ $current_page = $status['processed_pages'] ?? 0;
	133	+ $total_pages = $status['total_pages'] ?? 0;
	134	+
	135	+ if ($current_page >= $total_pages) {
	136	+ //error_log('Manual PDF: Already completed');
	137	+ return 0;
	138	+ }
	139	+
	140	+ // Try to download the PDF again for processing
	141	+ $response = wp_remote_get($pdf_url, array('timeout' => 30));
	142	+
	143	+ if (is_wp_error($response)) {
	144	+ //error_log('Manual PDF: Failed to download PDF: ' . $response->get_error_message());
	145	+ return 0;
	146	+ }
	147	+
	148	+ $pdf_content = wp_remote_retrieve_body($response);
	149	+ if (empty($pdf_content)) {
	150	+ //error_log('Manual PDF: Empty PDF content');
	151	+ return 0;
	152	+ }
	153	+
	154	+ // Save PDF temporarily
	155	+ $upload_dir = wp_upload_dir();
	156	+ $temp_pdf_path = trailingslashit($upload_dir['path']) . 'temp_manual_' . time() . '.pdf';
	157	+ file_put_contents($temp_pdf_path, $pdf_content);
	158	+
	159	+ // Process 2 pages directly
	160	+ $processed = $this->mxchat_process_pdf_pages_direct($temp_pdf_path, $pdf_url, $current_page, 5);
	161	+
	162	+ // Clean up temp file
	163	+ if (file_exists($temp_pdf_path)) {
	164	+ wp_delete_file($temp_pdf_path);
	165	+ }
	166	+
	167	+ //error_log('Manual PDF: Processed ' . $processed . ' pages');
	168	+ return $processed;
	169	+
	170	+ } catch (Exception $e) {
	171	+ //error_log('Manual PDF batch error: ' . $e->getMessage());
	172	+ return 0;
	173	+ }
	174	+}
	175	+
	176	+/**
	177	+ * Process PDF pages directly without cron
	178	+ */
	179	+private function mxchat_process_pdf_pages_direct($pdf_path, $pdf_url, $start_page, $batch_size) {
	180	+ try {
	181	+ if (!file_exists($pdf_path)) {
	182	+ //error_log('Direct PDF: File not found at ' . $pdf_path);
	183	+ return 0;
	184	+ }
	185	+
	186	+ $parser = new \Smalot\PdfParser\Parser();
	187	+ $pdf = $parser->parseFile($pdf_path);
	188	+ $pages = $pdf->getPages();
	189	+
	190	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	191	+ $status = get_transient($status_key);
	192	+
	193	+ if (!$status) {
	194	+ return 0;
	195	+ }
	196	+
	197	+ $options = get_option('mxchat_options');
	198	+ $api_key = $options['api_key'] ?? '';
	199	+
	200	+ if (empty($api_key)) {
	201	+ //error_log('Direct PDF: No API key');
	202	+ return 0;
	203	+ }
	204	+
	205	+ $processed = 0;
	206	+ $end_page = min($start_page + $batch_size, count($pages));
	207	+
	208	+ for ($i = $start_page; $i < $end_page; $i++) {
	209	+ try {
	210	+ $page_number = $i + 1;
	211	+ $text = $pages[$i]->getText();
	212	+
	213	+ if (empty($text)) {
	214	+ //error_log('Direct PDF: Empty text on page ' . $page_number);
	215	+ continue;
	216	+ }
	217	+
	218	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
	219	+ if (empty($sanitized_content)) {
	220	+ //error_log('Direct PDF: No content after sanitization on page ' . $page_number);
	221	+ continue;
	222	+ }
	223	+
	224	+ // Generate embedding
	225	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	226	+ if (is_string($embedding_vector)) {
	227	+ //error_log('Direct PDF: Embedding failed on page ' . $page_number . ': ' . $embedding_vector);
	228	+ continue;
	229	+ }
	230	+
	231	+ // Create metadata
	232	+ $metadata = array(
	233	+ 'document_type' => 'pdf',
	234	+ 'total_pages' => count($pages),
	235	+ 'current_page' => $page_number,
	236	+ 'source_url' => $pdf_url
	237	+ );
	238	+
	239	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	240	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
	241	+
	242	+ // Store in database
	243	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $api_key);
	244	+
	245	+ if (is_wp_error($db_result)) {
	246	+ //error_log('Direct PDF: DB error on page ' . $page_number . ': ' . $db_result->get_error_message());
	247	+ continue;
	248	+ }
	249	+
	250	+ $processed++;
	251	+ //error_log('Direct PDF: Successfully processed page ' . $page_number);
	252	+
	253	+ // Update status
	254	+ $status['processed_pages'] = $i + 1;
	255	+ $status['last_update'] = time();
	256	+ $status['percentage'] = round(($status['processed_pages'] / $status['total_pages']) * 100);
	257	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	258	+
	259	+ } catch (Exception $e) {
	260	+ //error_log('Direct PDF: Error processing page ' . ($i + 1) . ': ' . $e->getMessage());
	261	+ continue;
	262	+ }
	263	+ }
	264	+
	265	+ // Check if completed
	266	+ if ($status['processed_pages'] >= $status['total_pages']) {
	267	+ $status['status'] = 'complete';
	268	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	269	+ //error_log('Direct PDF: Processing completed');
	270	+ }
	271	+
	272	+ return $processed;
	273	+
	274	+ } catch (Exception $e) {
	275	+ //error_log('Direct PDF processing error: ' . $e->getMessage());
	276	+ return 0;
	277	+ }
	278	+}
	279	+
	280	+/**
	281	+ * Process a small sitemap batch manually - DIRECT PROCESSING
	282	+ */
	283	+private function mxchat_manual_process_sitemap_batch($sitemap_url) {
	284	+ try {
	285	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	286	+ $status = get_transient($status_key);
	287	+
	288	+ if (!$status || $status['status'] !== 'processing') {
	289	+ return 0;
	290	+ }
	291	+
	292	+ //error_log('Manual Sitemap: Starting direct processing for ' . $sitemap_url);
	293	+
	294	+ // Re-fetch the sitemap to get URLs
	295	+ $response = wp_remote_get($sitemap_url, array('timeout' => 30));
	296	+ if (is_wp_error($response)) {
	297	+ //error_log('Manual Sitemap: Failed to fetch sitemap');
	298	+ return 0;
	299	+ }
	300	+
	301	+ $sitemap_content = wp_remote_retrieve_body($response);
	302	+ $xml = simplexml_load_string($sitemap_content);
	303	+
	304	+ if (!$xml) {
	305	+ //error_log('Manual Sitemap: Invalid XML');
	306	+ return 0;
	307	+ }
	308	+
	309	+ $urls = array();
	310	+ foreach ($xml->url as $url_element) {
	311	+ $urls[] = (string)$url_element->loc;
	312	+ }
	313	+
	314	+ $current_processed = $status['processed_urls'] ?? 0;
	315	+ $batch_size = 5;
	316	+ $processed = 0;
	317	+
	318	+ // Process next 2 URLs
	319	+ for ($i = $current_processed; $i < min($current_processed + $batch_size, count($urls)); $i++) {
	320	+ $url = $urls[$i];
	321	+
	322	+ if ($this->mxchat_process_single_url_direct($url)) {
	323	+ $processed++;
	324	+ }
	325	+
	326	+ // Update status
	327	+ $status['processed_urls'] = $i + 1;
	328	+ $status['last_update'] = time();
	329	+ $status['percentage'] = round(($status['processed_urls'] / $status['total_urls']) * 100);
	330	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	331	+ }
	332	+
	333	+ // Check if completed
	334	+ if ($status['processed_urls'] >= $status['total_urls']) {
	335	+ $status['status'] = 'complete';
	336	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	337	+ }
	338	+
	339	+ //error_log('Manual Sitemap: Processed ' . $processed . ' URLs');
	340	+ return $processed;
	341	+
	342	+ } catch (Exception $e) {
	343	+ //error_log('Manual sitemap batch error: ' . $e->getMessage());
	344	+ return 0;
	345	+ }
	346	+}
	347	+
	348	+/**
	349	+ * Process a single URL directly
	350	+ */
	351	+private function mxchat_process_single_url_direct($url) {
	352	+ try {
	353	+ $response = wp_remote_get($url, array('timeout' => 30));
	354	+ if (is_wp_error($response)) {
	355	+ return false;
	356	+ }
	357	+
	358	+ $html = wp_remote_retrieve_body($response);
	359	+ $content = $this->mxchat_extract_main_content($html);
	360	+ $sanitized = $this->mxchat_sanitize_content_for_api($content);
	361	+
	362	+ if (empty($sanitized)) {
	363	+ return false;
	364	+ }
	365	+
	366	+ $options = get_option('mxchat_options');
	367	+ $api_key = $options['api_key'] ?? '';
	368	+
	369	+ $result = MxChat_Utils::submit_content_to_db($sanitized, $url, $api_key);
	370	+
	371	+ return !is_wp_error($result);
	372	+
	373	+ } catch (Exception $e) {
	374	+ //error_log('Single URL processing error: ' . $e->getMessage());
	375	+ return false;
	376	+ }
	377	+}
	378	+
	379	+
118	380	// ========================================
119	381	// MAIN CONTENT SUBMISSION HANDLERS
120	382	// ========================================
121	383
		@@ -131,30 +393,34 @@
131	393	wp_die(esc_html__('Nonce verification failed.', 'mxchat'));
132	394	}
133	395
134	396	// Sanitize the inputs.
135		- // Use wp_kses_post to allow safe HTML and wp_unslash to remove WordPress added slashes
136		- $article_content = wp_kses_post(wp_unslash($_POST['article_content']));
	397	+ $article_content = sanitize_textarea_field($_POST['article_content']);
137	398	$article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
138	399
139		- // Get bot_id from form submission
140		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
	400	+ // Get API key for submission
	401	+ $options = get_option('mxchat_options');
	402	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
141	403
142		- // Get bot-specific options and API key
143		- $bot_options = $this->get_bot_options($bot_id);
144		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
145		-
146		- // Custom-provider-aware decision; keyless custom sites must pass (plan cbd5fd).
147		- $preflight = MxChat_Utils::embedding_preflight($options);
148		- if (!$preflight['ok']) {
149		- set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
	404	+ if (strpos($selected_model, 'voyage') === 0) {
	405	+ $api_key = $options['voyage_api_key'] ?? '';
	406	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	407	+ $api_key = $options['gemini_api_key'] ?? '';
	408	+ } else {
	409	+ $api_key = $options['api_key'] ?? '';
	410	+ }
	411	+
	412	+ if (empty($api_key)) {
	413	+ set_transient('mxchat_admin_notice_error',
	414	+ esc_html__('API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
	415	+ 30
	416	+ );
150	417	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
151	418	exit;
152	419	}
153		- $api_key = $preflight['api_key'];
154	420
155		- // Use centralized utility function with bot_id
156		- $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key, null, $bot_id);
	421	+ // Use centralized utility function for storage
	422	+ $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key);
157	423
158	424	if (is_wp_error($result)) {
159	425	set_transient('mxchat_admin_notice_error',
160	426	esc_html__('Error storing content: ', 'mxchat') . $result->get_error_message(),
		@@ -169,253 +435,17 @@
169	435
170	436	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
171	437	exit;
172	438	}
173		-
174		-/**
175		- * Handle the "YouTube" KB import source (admin-post form submission).
176		- *
177		- * Per-video description mode:
178		- * - auto: fetch oEmbed metadata (reliable) + best-effort captions transcript.
179		- * If no usable transcript, index the metadata anyway, tell the admin,
180		- * and bounce back with the manual box pre-filled (never fail silently).
181		- * - manual: the admin's own description is what gets indexed; metadata rides along.
182		- *
183		- * The row is stored with content_type 'youtube' and source_url = the canonical
184		- * watch URL, so re-importing the same video UPDATES the entry (source_url
185		- * duplicate handling in MxChat_Utils::store_in_wordpress_db) — that is also the
186		- * "augment a metadata-only entry" path.
187		- */
188		-public function mxchat_handle_youtube_submission() {
189		- if (!isset($_POST['submit_youtube']) || !current_user_can('manage_options')) {
190		- wp_die(esc_html__('Unauthorized access', 'mxchat'));
191		- }
192		-
193		- check_admin_referer('mxchat_submit_youtube_action', 'mxchat_submit_youtube_nonce');
194		-
195		- $redirect_url = admin_url('admin.php?page=mxchat-prompts');
196		-
197		- $youtube_url = isset($_POST['youtube_url']) ? esc_url_raw(wp_unslash($_POST['youtube_url'])) : '';
198		- $video_id = MxChat_Utils::parse_youtube_id($youtube_url);
199		-
200		- if (empty($video_id)) {
201		- set_transient('mxchat_admin_notice_error',
202		- esc_html__('That does not look like a link to a single YouTube video. Please paste a watch, youtu.be, or Shorts URL.', 'mxchat'),
203		- 30
204		- );
205		- wp_safe_redirect(esc_url($redirect_url));
206		- exit;
207		- }
208		-
209		- $canonical_url = 'https://www.youtube.com/watch?v=' . $video_id;
210		-
211		- $description_mode = (isset($_POST['youtube_description_mode']) && $_POST['youtube_description_mode'] === 'manual') ? 'manual' : 'auto';
212		- $manual_description = isset($_POST['youtube_description']) ? trim(wp_kses_post(wp_unslash($_POST['youtube_description']))) : '';
213		-
214		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
215		-
216		- // Resolve the embedding decision exactly like the sibling handlers —
217		- // custom-provider-aware (plan cbd5fd).
218		- $bot_options = $this->get_bot_options($bot_id);
219		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
220		-
221		- $preflight = MxChat_Utils::embedding_preflight($options);
222		- if (!$preflight['ok']) {
223		- set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
224		- wp_safe_redirect(esc_url($redirect_url));
225		- exit;
226		- }
227		- $api_key = $preflight['api_key'];
228		-
229		- // Metadata is fetched in BOTH modes — it is the reliable half of auto, and in
230		- // manual mode it enriches the indexed text with the real title/channel.
231		- $meta = $this->mxchat_fetch_youtube_oembed($video_id);
232		- $video_title = isset($meta['title']) ? sanitize_text_field($meta['title']) : '';
233		- $video_channel = isset($meta['author_name']) ? sanitize_text_field($meta['author_name']) : '';
234		-
235		- $header_lines = 'YouTube Video: ' . ($video_title !== '' ? $video_title : $canonical_url) . "\n";
236		- if ($video_channel !== '') {
237		- $header_lines .= 'Channel: ' . $video_channel . "\n";
238		- }
239		- $header_lines .= 'URL: ' . $canonical_url . "\n\n";
240		-
241		- $transcript_missing = false;
242		-
243		- if ($description_mode === 'manual') {
244		- if ($manual_description === '') {
245		- set_transient('mxchat_admin_notice_error',
246		- esc_html__('Please write a description for the video, or switch to auto-fetch.', 'mxchat'),
247		- 30
248		- );
249		- wp_safe_redirect(esc_url($redirect_url));
250		- exit;
251		- }
252		- $indexed_text = $header_lines . $manual_description;
253		- } else {
254		- $transcript = $this->mxchat_fetch_youtube_transcript($video_id);
255		-
256		- if (strlen($transcript) >= 200) {
257		- $indexed_text = $header_lines . $transcript;
258		- } else {
259		- // Graceful fallback: captions disabled / blocked / no speech. Auto
260		- // reliably gets metadata; it does NOT guarantee a transcript.
261		- $transcript_missing = true;
262		-
263		- if ($video_title === '' && $video_channel === '') {
264		- // Both halves failed — nothing meaningful to index.
265		- set_transient('mxchat_admin_notice_error',
266		- esc_html__('Could not retrieve any information for that video (no metadata and no captions). Please check the URL, or use the manual description option.', 'mxchat'),
267		- 30
268		- );
269		- wp_safe_redirect(esc_url($redirect_url));
270		- exit;
271		- }
272		-
273		- $indexed_text = $header_lines . sprintf(
274		- /* translators: 1: video title, 2: channel name */
275		- __('A YouTube video titled "%1$s" from the channel %2$s.', 'mxchat'),
276		- $video_title !== '' ? $video_title : $canonical_url,
277		- $video_channel !== '' ? $video_channel : 'YouTube'
278		- );
279		- }
280		- }
281		-
282		- $result = MxChat_Utils::submit_content_to_db($indexed_text, $canonical_url, $api_key, null, $bot_id, 'youtube');
283		-
284		- if (is_wp_error($result)) {
285		- set_transient('mxchat_admin_notice_error',
286		- esc_html__('Error storing video in the knowledge base: ', 'mxchat') . $result->get_error_message(),
287		- 30
288		- );
289		- wp_safe_redirect(esc_url($redirect_url));
290		- exit;
291		- }
292		-
293		- if ($transcript_missing) {
294		- set_transient('mxchat_admin_notice_success',
295		- esc_html__('Video indexed from its title and channel — no captions were available for a transcript. The form below is pre-filled: write your own description and import again to improve matching (it updates the same entry).', 'mxchat'),
296		- 30
297		- );
298		- // Bounce back with prefill args so the page reopens the YouTube form in
299		- // manual mode with the URL + fetched title ready to augment.
300		- $redirect_url = add_query_arg(array(
301		- 'mxchat_yt_prefill' => '1',
302		- 'yt_url' => rawurlencode($canonical_url),
303		- 'yt_title' => rawurlencode($video_title),
304		- ), $redirect_url);
305		- } else {
306		- set_transient('mxchat_admin_notice_success',
307		- esc_html__('YouTube video successfully added to the knowledge base!', 'mxchat'),
308		- 30
309		- );
310		- }
311		-
312		- wp_safe_redirect(esc_url_raw($redirect_url));
313		- exit;
314		-}
315		-
316		-/**
317		- * Fetch YouTube oEmbed metadata for a video (no API key required).
318		- * Returns the decoded array (title, author_name, thumbnail_url, ...) or array().
319		- */
320		-private function mxchat_fetch_youtube_oembed($video_id) {
321		- $oembed_url = 'https://www.youtube.com/oembed?url=' . rawurlencode('https://www.youtube.com/watch?v=' . $video_id) . '&format=json';
322		- $response = wp_remote_get($oembed_url, array('timeout' => 15));
323		- if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
324		- return array();
325		- }
326		- $data = json_decode(wp_remote_retrieve_body($response), true);
327		- return is_array($data) ? $data : array();
328		-}
329		-
330		-/**
331		- * Best-effort captions transcript for a video. Deliberately ISOLATED: this uses
332		- * YouTube's unofficial timedtext route (the caption track list embedded in the
333		- * watch page), which YouTube has broken before and will break again. Every
334		- * failure mode returns '' so a break degrades to the metadata-only import path
335		- * instead of erroring the whole submission. Do not let anything in here throw.
336		- */
337		-private function mxchat_fetch_youtube_transcript($video_id) {
338		- $watch_url = 'https://www.youtube.com/watch?v=' . $video_id . '&hl=en';
339		-
340		- // First try the honest ingest UA; some responses omit the player config for
341		- // bot UAs, so retry once with a browser UA before giving up.
342		- $user_agents = array(
343		- mxchat_ingest_user_agent(),
344		- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
345		- );
346		-
347		- $tracks = array();
348		- foreach ($user_agents as $ua) {
349		- $response = wp_remote_get($watch_url, array(
350		- 'timeout' => 20,
351		- 'user-agent' => $ua,
352		- ));
353		- if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
354		- continue;
355		- }
356		- $body = wp_remote_retrieve_body($response);
357		- if (!is_string($body) || $body === '' || !preg_match('/"captionTracks":(\[.*?\])(?=,")/s', $body, $m)) {
358		- continue;
359		- }
360		- $decoded = json_decode($m[1], true);
361		- if (is_array($decoded) && !empty($decoded)) {
362		- $tracks = $decoded;
363		- break;
364		- }
365		- }
366		-
367		- if (empty($tracks)) {
368		- return '';
369		- }
370		-
371		- // Prefer an English track, else take the first offered.
372		- $chosen = null;
373		- foreach ($tracks as $track) {
374		- if (isset($track['languageCode']) && strpos($track['languageCode'], 'en') === 0) {
375		- $chosen = $track;
376		- break;
377		- }
378		- }
379		- if ($chosen === null) {
380		- $chosen = $tracks[0];
381		- }
382		- if (empty($chosen['baseUrl']) || !is_string($chosen['baseUrl'])) {
383		- return '';
384		- }
385		-
386		- $timedtext = wp_remote_get($chosen['baseUrl'], array('timeout' => 20));
387		- if (is_wp_error($timedtext) || wp_remote_retrieve_response_code($timedtext) !== 200) {
388		- return '';
389		- }
390		- $xml = wp_remote_retrieve_body($timedtext);
391		- if (!is_string($xml) || strpos($xml, '<text') === false) {
392		- return '';
393		- }
394		-
395		- // <text start=".." dur="..">caption</text> — strip tags, decode the
396		- // double-encoded entities timedtext ships, collapse whitespace.
397		- $text = preg_replace('/<[^>]+>/', ' ', $xml);
398		- $text = html_entity_decode(html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8'), ENT_QUOTES | ENT_HTML5, 'UTF-8');
399		- $text = trim(preg_replace('/\s+/u', ' ', $text));
400		-
401		- return $text;
402		-}
403		-
404	439	public function mxchat_is_pdf_url($url, $response) {
405	440	$content_type = wp_remote_retrieve_header($response, 'content-type');
406	441	$file_extension = strtolower(pathinfo($url, PATHINFO_EXTENSION));
407	442
408		- // Check Content-Disposition header for .pdf filename (Google Drive sends this)
409		- $disposition = wp_remote_retrieve_header($response, 'content-disposition');
410		- $has_pdf_disposition = ! empty($disposition) && stripos($disposition, '.pdf') !== false;
411		-
412		- return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf' || $has_pdf_disposition;
	443	+ return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf';
413	444	}
414		-
415		-
416		-public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response, $bot_id = 'default') {
	445	+public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response) {
417	446	if (!current_user_can('manage_options')) {
	447	+ //error_log(esc_html__('Unauthorized PDF processing attempt', 'mxchat'));
418	448	return false;
419	449	}
420	450
421	451	$pdf_url = esc_url_raw($pdf_url);
		@@ -421,8 +451,9 @@
421	451	$pdf_url = esc_url_raw($pdf_url);
422	452	$upload_dir = wp_upload_dir();
423	453
424	454	if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
	455	+ //error_log(sprintf(esc_html__('Upload directory error: %s', 'mxchat'), esc_html($upload_dir['error'])));
425	456	return false;
426	457	}
427	458
428	459	$pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
		@@ -429,12 +460,14 @@
429	460	$pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
430	461
431	462	$response_body = wp_remote_retrieve_body($response);
432	463	if (empty($response_body)) {
	464	+ //error_log(esc_html__('Empty PDF response body', 'mxchat'));
433	465	return false;
434	466	}
435	467
436	468	if (!wp_mkdir_p(dirname($pdf_path))) {
	469	+ //error_log(sprintf(esc_html__('Failed to create directory for PDF: %s', 'mxchat'), esc_html($pdf_path)));
437	470	return false;
438	471	}
439	472
440	473	try {
		@@ -443,915 +476,303 @@
443	476	if (!file_exists($pdf_path)) {
444	477	throw new Exception(__('Failed to save PDF file', 'mxchat'));
445	478	}
446	479
447		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
448		-
449		- if ($total_pages === false || $total_pages < 1) {
450		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
451		- }
	480	+ $parser = new \Smalot\PdfParser\Parser();
	481	+ $pdf = $parser->parseFile($pdf_path);
	482	+ $total_pages = absint(count($pdf->getPages()));
452	483
453		- // Create unique queue ID
454		- $queue_id = 'pdf_' . md5($pdf_url . time());
455		-
456		- // Create array of pages to process
457		- $pages = array();
458		- for ($i = 1; $i <= $total_pages; $i++) {
459		- $pages[] = array(
460		- 'pdf_path' => $pdf_path,
461		- 'pdf_url' => $pdf_url,
462		- 'page_number' => $i,
463		- 'total_pages' => $total_pages
464		- );
	484	+ if ($total_pages < 1) {
	485	+ throw new Exception(__('Invalid PDF: no pages found', 'mxchat'));
465	486	}
466	487
467		- // Add pages to queue
468		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
	488	+ wp_schedule_single_event(time(), 'mxchat_process_pdf_pages', array(
	489	+ 'pdf_path' => $pdf_path,
	490	+ 'pdf_url' => $pdf_url,
	491	+ 'total_pages' => $total_pages,
	492	+ 'batch_size' => absint(15),
	493	+ 'batch_pause' => absint(10)
	494	+ ));
469	495
470		- if ($queued_count === 0) {
471		- wp_delete_file($pdf_path);
472		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
473		- }
	496	+ $status_data = array(
	497	+ 'total_pages' => $total_pages,
	498	+ 'processed_pages' => 0,
	499	+ 'status' => 'processing',
	500	+ 'last_update' => time()
	501	+ );
474	502
475		- // Store queue metadata
476		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $pdf_url);
477		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
478		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
479		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
480		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
481		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	503	+ set_transient(
	504	+ sanitize_key('mxchat_pdf_status_' . md5($pdf_url)),
	505	+ array_map('sanitize_text_field', $status_data),
	506	+ DAY_IN_SECONDS
	507	+ );
482	508
483		- // Store queue ID in transient for status tracking
484		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
485		- set_transient('mxchat_last_pdf_url', $pdf_url, DAY_IN_SECONDS);
	509	+ return __('scheduled', 'mxchat');
486	510
487		- return 'queued';
488		-
489	511	} catch (Exception $e) {
	512	+ //error_log(sprintf(esc_html__('Error preparing PDF for processing: %s', 'mxchat'), esc_html($e->getMessage())));
490	513	if (file_exists($pdf_path)) {
491	514	wp_delete_file($pdf_path);
492	515	}
493		- return $e->getMessage();
	516	+ return false;
494	517	}
495	518	}
496	519
497		-/**
498		- * Handle direct PDF file upload from the knowledge base page
499		- */
500		-public function mxchat_handle_pdf_file_submission() {
501		- if (!isset($_POST['submit_pdf_file']) || !current_user_can('manage_options')) {
502		- wp_die(esc_html__('Unauthorized access', 'mxchat'));
503		- }
	520	+public function mxchat_process_pdf_pages_cron($pdf_path, $pdf_url, $total_pages, $batch_size, $batch_pause) {
	521	+ // Validate inputs
	522	+ $pdf_path = sanitize_text_field($pdf_path);
	523	+ $pdf_url = esc_url_raw($pdf_url);
	524	+ $total_pages = absint($total_pages);
	525	+ $batch_size = absint($batch_size);
	526	+ $batch_pause = absint($batch_pause);
504	527
505		- check_admin_referer('mxchat_submit_pdf_file_action', 'mxchat_submit_pdf_file_nonce');
506		-
507		- $redirect_url = admin_url('admin.php?page=mxchat-prompts');
508		-
509		- // Validate file upload
510		- if (empty($_FILES['pdf_file']) || $_FILES['pdf_file']['error'] !== UPLOAD_ERR_OK) {
511		- $error_code = isset($_FILES['pdf_file']['error']) ? $_FILES['pdf_file']['error'] : UPLOAD_ERR_NO_FILE;
512		- $error_messages = array(
513		- UPLOAD_ERR_INI_SIZE => __('The uploaded file exceeds the server upload_max_filesize limit.', 'mxchat'),
514		- UPLOAD_ERR_FORM_SIZE => __('The uploaded file exceeds the form MAX_FILE_SIZE limit.', 'mxchat'),
515		- UPLOAD_ERR_PARTIAL => __('The file was only partially uploaded.', 'mxchat'),
516		- UPLOAD_ERR_NO_FILE => __('No file was uploaded. Please select a PDF file.', 'mxchat'),
517		- UPLOAD_ERR_NO_TMP_DIR => __('Server missing temporary folder.', 'mxchat'),
518		- UPLOAD_ERR_CANT_WRITE => __('Server failed to write file to disk.', 'mxchat'),
519		- );
520		- $error_msg = isset($error_messages[$error_code]) ? $error_messages[$error_code] : __('Unknown upload error.', 'mxchat');
521		- set_transient('mxchat_admin_notice_error', $error_msg, 30);
522		- wp_safe_redirect(esc_url($redirect_url));
523		- exit;
524		- }
525		-
526		- $file = $_FILES['pdf_file'];
527		-
528		- // Validate MIME type
529		- $finfo = finfo_open(FILEINFO_MIME_TYPE);
530		- $mime_type = finfo_file($finfo, $file['tmp_name']);
531		- finfo_close($finfo);
532		-
533		- if ($mime_type !== 'application/pdf') {
534		- set_transient('mxchat_admin_notice_error',
535		- esc_html__('Invalid file type. Only PDF files are accepted.', 'mxchat'),
536		- 30
537		- );
538		- wp_safe_redirect(esc_url($redirect_url));
539		- exit;
540		- }
541		-
542		- // Validate extension
543		- $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
544		- if ($ext !== 'pdf') {
545		- set_transient('mxchat_admin_notice_error',
546		- esc_html__('Invalid file extension. Only .pdf files are accepted.', 'mxchat'),
547		- 30
548		- );
549		- wp_safe_redirect(esc_url($redirect_url));
550		- exit;
551		- }
552		-
553		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
554		- $original_filename = sanitize_file_name($file['name']);
555		-
556		- $upload_dir = wp_upload_dir();
557		- if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
558		- set_transient('mxchat_admin_notice_error',
559		- esc_html__('WordPress upload directory is not writable.', 'mxchat'),
560		- 30
561		- );
562		- wp_safe_redirect(esc_url($redirect_url));
563		- exit;
564		- }
565		-
566		- $pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
567		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
568		-
569		- if (!wp_mkdir_p(dirname($pdf_path))) {
570		- set_transient('mxchat_admin_notice_error',
571		- esc_html__('Failed to create upload directory.', 'mxchat'),
572		- 30
573		- );
574		- wp_safe_redirect(esc_url($redirect_url));
575		- exit;
576		- }
577		-
578		- // Move uploaded file
579		- if (!move_uploaded_file($file['tmp_name'], $pdf_path)) {
580		- set_transient('mxchat_admin_notice_error',
581		- esc_html__('Failed to save uploaded PDF file.', 'mxchat'),
582		- 30
583		- );
584		- wp_safe_redirect(esc_url($redirect_url));
585		- exit;
586		- }
587		-
588	528	try {
589		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
590		-
591		- if ($total_pages === false || $total_pages < 1) {
592		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
	529	+ if (!file_exists($pdf_path)) {
	530	+ throw new Exception(sprintf('PDF file not found at path: %s', esc_html($pdf_path)));
593	531	}
594	532
595		- // Use original filename as the source identifier
596		- $source_label = 'upload://' . $original_filename;
	533	+ $parser = new \Smalot\PdfParser\Parser();
	534	+ $pdf = $parser->parseFile($pdf_path);
	535	+ $pages = $pdf->getPages();
597	536
598		- $queue_id = 'pdf_' . md5($source_label . time());
	537	+ // Get current progress
	538	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	539	+ $status = get_transient($status_key);
599	540
600		- $pages = array();
601		- for ($i = 1; $i <= $total_pages; $i++) {
602		- $pages[] = array(
603		- 'pdf_path' => $pdf_path,
604		- 'pdf_url' => $source_label,
605		- 'page_number' => $i,
606		- 'total_pages' => $total_pages,
607		- );
	541	+ if (!$status || !is_array($status)) {
	542	+ throw new Exception('Invalid status data retrieved from transient');
608	543	}
609	544
610		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
	545	+ // Initialize failed pages list if it doesn't exist
	546	+ if (!isset($status['failed_pages_list']) || !is_array($status['failed_pages_list'])) {
	547	+ $status['failed_pages_list'] = [];
	548	+ }
611	549
612		- if ($queued_count === 0) {
613		- wp_delete_file($pdf_path);
614		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
	550	+ $start_page = absint($status['processed_pages']);
	551	+ $end_page = min($start_page + $batch_size, $total_pages);
	552	+ $options = get_option('mxchat_options');
	553	+
	554	+ if (empty($options['api_key'])) {
	555	+ throw new Exception('API key is missing or invalid');
615	556	}
616	557
617		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $source_label);
618		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
619		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
620		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
621		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
622		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	558	+ $successful_pages = 0;
	559	+ $failed_pages = 0;
623	560
624		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
625		- set_transient('mxchat_last_pdf_url', $source_label, DAY_IN_SECONDS);
	561	+ for ($i = $start_page; $i < $end_page; $i++) {
	562	+ $page_number = $i + 1;
	563	+ $max_retries = 3;
	564	+ $retry_count = 0;
	565	+ $page_processed = false;
	566	+ $last_error = '';
626	567
627		- set_transient('mxchat_admin_notice_success',
628		- sprintf(
629		- esc_html__('PDF "%s" (%d pages) queued for processing. Processing will start automatically.', 'mxchat'),
630		- esc_html($original_filename),
631		- $total_pages
632		- ),
633		- 30
634		- );
	568	+ while (!$page_processed && $retry_count < $max_retries) {
	569	+ try {
	570	+ $text = $pages[$i]->getText();
	571	+
	572	+ if (empty($text)) {
	573	+ throw new Exception("Empty text on page {$page_number}");
	574	+ }
	575	+
	576	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
635	577
636		- } catch (Exception $e) {
637		- if (file_exists($pdf_path)) {
638		- wp_delete_file($pdf_path);
639		- }
640		- set_transient('mxchat_admin_notice_error',
641		- esc_html__('Failed to process uploaded PDF: ', 'mxchat') . esc_html($e->getMessage()),
642		- 30
643		- );
644		- }
	578	+ if (empty($sanitized_content)) {
	579	+ throw new Exception("No valid content after sanitization on page {$page_number}");
	580	+ }
645	581
646		- wp_safe_redirect(esc_url($redirect_url));
647		- exit;
648		-}
	582	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	583	+
	584	+ if (is_string($embedding_vector)) {
	585	+ throw new Exception("Embedding generation failed: " . $embedding_vector);
	586	+ }
	587	+
	588	+ if (!is_array($embedding_vector)) {
	589	+ throw new Exception("Embedding generation returned unexpected result type: " . gettype($embedding_vector));
	590	+ }
	591	+
	592	+ $metadata = array(
	593	+ 'document_type' => 'pdf',
	594	+ 'total_pages' => $total_pages,
	595	+ 'current_page' => $page_number,
	596	+ 'prev_page' => $i > 0 ? $i : null,
	597	+ 'next_page' => $i < ($total_pages - 1) ? $i + 2 : null,
	598	+ 'source_url' => $pdf_url
	599	+ );
649	600
650		-/**
651		- * Validate PDF and count pages with multiple parser attempts
652		- */
653		-private function mxchat_validate_and_count_pdf_pages($pdf_path) {
654		- // Method 1: Try with Smalot PDF Parser (your current method)
655		- try {
656		- mxchat_load_pdf_parser();
657		- $parser = new \Smalot\PdfParser\Parser();
658		- $pdf = $parser->parseFile($pdf_path);
659		- $pages = $pdf->getPages();
660		- $page_count = count($pages);
661		-
662		- if ($page_count > 0) {
663		- //error_log('PDF parsed successfully with Smalot parser: ' . $page_count . ' pages');
664		- return $page_count;
665		- }
666		- } catch (Exception $e) {
667		- //error_log('Smalot PDF parser failed: ' . $e->getMessage());
668		- }
	601	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	602	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
669	603
670		- // Method 2: Try with pdfinfo command (if available)
671		- if (function_exists('shell_exec') && !$this->mxchat_is_shell_disabled()) {
672		- try {
673		- $command = 'pdfinfo ' . escapeshellarg($pdf_path) . ' 2>&1';
674		- $output = shell_exec($command);
675		-
676		- if ($output && preg_match('/Pages:\s*(\d+)/', $output, $matches)) {
677		- $page_count = intval($matches[1]);
678		- if ($page_count > 0) {
679		- //error_log('PDF parsed successfully with pdfinfo: ' . $page_count . ' pages');
680		- return $page_count;
	604	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $options['api_key']);
	605	+
	606	+ if (is_wp_error($db_result)) {
	607	+ throw new Exception("Database submission failed: " . $db_result->get_error_message());
	608	+ }
	609	+
	610	+ // Success!
	611	+ $page_processed = true;
	612	+ $successful_pages++;
	613	+
	614	+ } catch (Exception $e) {
	615	+ $retry_count++;
	616	+ $last_error = $e->getMessage();
	617	+
	618	+ //error_log("PDF page {$page_number} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	619	+
	620	+ if ($retry_count < $max_retries) {
	621	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	622	+ sleep(pow(2, $retry_count - 1));
	623	+ }
681	624	}
682	625	}
683		- } catch (Exception $e) {
684		- //error_log('pdfinfo command failed: ' . $e->getMessage());
685		- }
686		- }
687	626
688		- // Method 3: Try to repair PDF and parse again
689		- try {
690		- $repaired_path = $this->mxchat_attempt_pdf_repair($pdf_path);
691		- if ($repaired_path && $repaired_path !== $pdf_path) {
692		- mxchat_load_pdf_parser();
693		- $parser = new \Smalot\PdfParser\Parser();
694		- $pdf = $parser->parseFile($repaired_path);
695		- $pages = $pdf->getPages();
696		- $page_count = count($pages);
697		-
698		- if ($page_count > 0) {
699		- // Replace original with repaired version
700		- copy($repaired_path, $pdf_path);
701		- unlink($repaired_path);
702		- //error_log('PDF repaired and parsed successfully: ' . $page_count . ' pages');
703		- return $page_count;
	627	+ // If page still not processed after all retries, mark as failed
	628	+ if (!$page_processed) {
	629	+ $failed_pages++;
	630	+ $status['failed_pages_list'][] = [
	631	+ 'page' => $page_number,
	632	+ 'error' => $last_error,
	633	+ 'time' => time(),
	634	+ 'retries' => $max_retries
	635	+ ];
	636	+
	637	+ // Limit failed pages list to prevent memory issues
	638	+ if (count($status['failed_pages_list']) > 50) {
	639	+ $status['failed_pages_list'] = array_slice($status['failed_pages_list'], -50);
	640	+ }
	641	+
	642	+ //error_log("PDF page {$page_number} permanently failed after {$max_retries} attempts: " . $last_error);
704	643	}
	644	+
	645	+ // Update progress
	646	+ $status['processed_pages'] = absint($page_number);
	647	+ $status['last_update'] = time();
	648	+ $status['failed_pages'] = absint($status['failed_pages'] ?? 0) + ($page_processed ? 0 : 1);
705	649
706		- // Clean up repaired file if it didn't work
707		- unlink($repaired_path);
	650	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
708	651	}
709		- } catch (Exception $e) {
710		- //error_log('PDF repair attempt failed: ' . $e->getMessage());
711		- }
712	652
713		- // Method 4: Manual PDF structure analysis (basic page count)
714		- try {
715		- $page_count = $this->mxchat_manual_pdf_page_count($pdf_path);
716		- if ($page_count > 0) {
717		- //error_log('PDF page count determined manually: ' . $page_count . ' pages');
718		- return $page_count;
719		- }
720		- } catch (Exception $e) {
721		- //error_log('Manual PDF analysis failed: ' . $e->getMessage());
722		- }
723		-
724		- //error_log('All PDF parsing methods failed for: ' . $pdf_path);
725		- return false;
726		-}
727		-
728		-/**
729		- * Check if shell_exec is disabled
730		- */
731		-private function mxchat_is_shell_disabled() {
732		- $disabled = explode(',', ini_get('disable_functions'));
733		- return in_array('shell_exec', $disabled);
734		-}
735		-
736		-/**
737		- * Attempt to repair PDF using basic methods
738		- */
739		-private function mxchat_attempt_pdf_repair($pdf_path) {
740		- try {
741		- $content = file_get_contents($pdf_path);
742		- if (!$content) {
743		- return false;
744		- }
745		-
746		- // Check if PDF starts with proper header
747		- if (substr($content, 0, 4) !== '%PDF') {
748		- // Try to find PDF header in the content
749		- $header_pos = strpos($content, '%PDF');
750		- if ($header_pos !== false && $header_pos < 1024) {
751		- // Remove junk before PDF header
752		- $content = substr($content, $header_pos);
753		- $repaired_path = $pdf_path . '.repaired';
754		- file_put_contents($repaired_path, $content);
755		- return $repaired_path;
	653	+ // Schedule next batch if needed
	654	+ if ($end_page < $total_pages) {
	655	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_pdf_pages', array(
	656	+ 'pdf_path' => $pdf_path,
	657	+ 'pdf_url' => $pdf_url,
	658	+ 'total_pages' => $total_pages,
	659	+ 'batch_size' => $batch_size,
	660	+ 'batch_pause' => $batch_pause
	661	+ ));
	662	+ } else {
	663	+ // Processing complete
	664	+ $status['status'] = 'complete';
	665	+ $status['processed_pages'] = $total_pages;
	666	+
	667	+ // Add completion summary
	668	+ $status['completion_summary'] = [
	669	+ 'total_pages' => $total_pages,
	670	+ 'successful_pages' => $total_pages - absint($status['failed_pages'] ?? 0),
	671	+ 'failed_pages' => absint($status['failed_pages'] ?? 0),
	672	+ 'completion_time' => current_time('mysql')
	673	+ ];
	674	+
	675	+ // Save the completed status (don't delete it - let user dismiss manually)
	676	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	677	+
	678	+ // Clean up the temporary PDF file
	679	+ if (file_exists($pdf_path)) {
	680	+ wp_delete_file($pdf_path);
756	681	}
	682	+
	683	+ // DON'T delete the status transients here - let user dismiss manually
757	684	}
758	685
759		- // Check for EOF marker
760		- $content = rtrim($content);
761		- if (!preg_match('/%%EOF\s*$/', $content)) {
762		- // Add EOF marker if missing
763		- $content .= "\n%%EOF";
764		- $repaired_path = $pdf_path . '.repaired';
765		- file_put_contents($repaired_path, $content);
766		- return $repaired_path;
	686	+ } catch (\Exception $e) {
	687	+ //error_log(sprintf('[MXCHAT-PDF] Error processing PDF: %s', $e->getMessage()));
	688	+
	689	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	690	+ $status = get_transient($status_key);
	691	+
	692	+ if (!$status || !is_array($status)) {
	693	+ $status = array(
	694	+ 'total_pages' => $total_pages,
	695	+ 'processed_pages' => 0,
	696	+ 'status' => 'error',
	697	+ 'error' => sanitize_text_field($e->getMessage()),
	698	+ 'last_update' => time()
	699	+ );
	700	+ } else {
	701	+ $status['status'] = 'error';
	702	+ $status['error'] = sanitize_text_field($e->getMessage());
	703	+ $status['last_update'] = time();
767	704	}
768		-
769		- } catch (Exception $e) {
770		- //error_log('PDF repair error: ' . $e->getMessage());
771		- }
772		-
773		- return false;
774		-}
775		-
776		-/**
777		- * Manual PDF page counting by analyzing PDF structure
778		- */
779		-private function mxchat_manual_pdf_page_count($pdf_path) {
780		- try {
781		- $content = file_get_contents($pdf_path);
782		- if (!$content) {
783		- return 0;
	705	+
	706	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	707	+
	708	+ if (file_exists($pdf_path)) {
	709	+ wp_delete_file($pdf_path);
784	710	}
785		-
786		- // Method 1: Count /Type /Page objects
787		- $page_count = preg_match_all('/\/Type\s*\/Page[^s]/', $content);
788		- if ($page_count > 0) {
789		- return $page_count;
790		- }
791		-
792		- // Method 2: Look for /Count in pages object
793		- if (preg_match('/\/Type\s*\/Pages.*?\/Count\s+(\d+)/', $content, $matches)) {
794		- return intval($matches[1]);
795		- }
796		-
797		- // Method 3: Count page references
798		- $page_count = preg_match_all('/\d+\s+0\s+obj\s*<<[^>]*\/Type\s*\/Page/', $content);
799		- if ($page_count > 0) {
800		- return $page_count;
801		- }
802		-
803		- } catch (Exception $e) {
804		- //error_log('Manual PDF analysis error: ' . $e->getMessage());
805	711	}
806		-
807		- return 0;
808	712	}
809	713
810		-
811	714	public function mxchat_save_inline_prompt() {
812	715	// DEBUG: Log what we're receiving
813		- //error_log('=== MXCHAT DEBUG ===');
814		- //error_log('POST data: ' . print_r($_POST, true));
815		- //error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
	716	+ error_log('=== MXCHAT DEBUG ===');
	717	+ error_log('POST data: ' . print_r($_POST, true));
	718	+ error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
816	719
817	720	// Check for nonce security
818	721	check_ajax_referer('mxchat_save_inline_nonce', '_ajax_nonce');
819	722
820	723	// If we get here, nonce passed
821		- //error_log('Nonce verification PASSED');
822		-
823		- // Verify permissions
824		- if (!current_user_can('manage_options')) {
825		- wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
826		- return;
827		- }
828		-
829		- global $wpdb;
830		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
831		-
832		- // Validate and sanitize input data
833		- $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
834		- $article_content = isset($_POST['article_content']) ? wp_kses_post(wp_unslash($_POST['article_content'])) : '';
835		- $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
836		-
837		- if ($prompt_id > 0 && !empty($article_content)) {
838		- // Re-generate the embedding vector for the updated content
839		- $embedding_vector = $this->mxchat_generate_embedding($article_content);
840		- if (is_array($embedding_vector)) {
841		- // Serialize the embedding vector before storing it
842		- $embedding_vector_serialized = serialize($embedding_vector);
843		- // Update the prompt in the database
844		- $updated = $wpdb->update(
845		- $table_name,
846		- array(
847		- 'article_content' => $article_content,
848		- 'embedding_vector' => $embedding_vector_serialized,
849		- 'source_url' => $article_url,
850		- ),
851		- array('id' => $prompt_id),
852		- array('%s', '%s', '%s'),
853		- array('%d')
854		- );
855		- if ($updated !== false) {
856		- wp_send_json_success();
857		- } else {
858		- MxChat_Admin::mxchat_log_debug('knowledge_error', 'Database update failed when saving knowledge entry');
859		- wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
860		- }
861		- } else {
862		- MxChat_Admin::mxchat_log_debug('embedding_error', 'Embedding generation failed for knowledge entry');
863		- wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
864		- }
865		- } else {
866		- wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
867		- }
868		-}
	724	+ error_log('Nonce verification PASSED');
869	725
	726	+ // Verify permissions
	727	+ if (!current_user_can('manage_options')) {
	728	+ wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
	729	+ return;
	730	+ }
870	731
871		-/**
872		- * AJAX: Get full content for editing — reassembles chunks if needed.
873		- * Works for both WordPress DB and Pinecone entries.
874		- */
875		-public function ajax_mxchat_get_entry_content() {
876		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
	732	+ global $wpdb;
	733	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
877	734
878		- if ( ! current_user_can('manage_options') ) {
879		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
880		- }
	735	+ // Validate and sanitize input data
	736	+ $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
	737	+ $article_content = isset($_POST['article_content']) ? sanitize_textarea_field($_POST['article_content']) : '';
	738	+ $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
881	739
882		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
883		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
884		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
885		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
	740	+ if ($prompt_id > 0 && !empty($article_content)) {
	741	+ // Re-generate the embedding vector for the updated content
	742	+ $embedding_vector = $this->mxchat_generate_embedding($article_content);
886	743
887		- if ( $data_source === 'pinecone' ) {
888		- // Pinecone: fetch vectors by source_url, reassemble chunks
889		- $content = $this->get_pinecone_entry_content( $source_url, $entry_id, $bot_id );
890		- } else {
891		- // WordPress DB
892		- $content = $this->get_wordpress_entry_content( $source_url, $entry_id );
893		- }
	744	+ if (is_array($embedding_vector)) {
	745	+ // Serialize the embedding vector before storing it
	746	+ $embedding_vector_serialized = serialize($embedding_vector);
894	747
895		- if ( is_wp_error( $content ) ) {
896		- wp_send_json_error( array( 'message' => $content->get_error_message() ) );
897		- }
	748	+ // Update the prompt in the database
	749	+ $updated = $wpdb->update(
	750	+ $table_name,
	751	+ array(
	752	+ 'article_content' => $article_content,
	753	+ 'embedding_vector' => $embedding_vector_serialized,
	754	+ 'source_url' => $article_url,
	755	+ ),
	756	+ array('id' => $prompt_id),
	757	+ array('%s', '%s', '%s'),
	758	+ array('%d')
	759	+ );
898	760
899		- wp_send_json_success( $content );
900		-}
	761	+ if ($updated !== false) {
	762	+ wp_send_json_success();
	763	+ } else {
	764	+ wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
	765	+ }
	766	+ } else {
	767	+ wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
	768	+ }
	769	+ } else {
	770	+ wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
	771	+ }
	772	+ }
901	773
902		-/**
903		- * Get content from WordPress DB — reassembles chunks by source_url.
904		- */
905		-private function get_wordpress_entry_content( $source_url, $entry_id ) {
906		- global $wpdb;
907		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
908	774
909		- // If we have a source_url, check for chunks
910		- if ( ! empty( $source_url ) && strpos( $source_url, 'mxchat://' ) !== 0 ) {
911		- $rows = $wpdb->get_results( $wpdb->prepare(
912		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
913		- $source_url
914		- ) );
915		-
916		- if ( $rows && count( $rows ) > 1 ) {
917		- // Multiple rows = chunked. Reassemble.
918		- $chunks = array();
919		- foreach ( $rows as $row ) {
920		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
921		- $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
922		- $chunks[ $index ] = $parsed['text'];
923		- }
924		- ksort( $chunks );
925		- return array(
926		- 'content' => implode( "\n\n", $chunks ),
927		- 'source_url' => $source_url,
928		- 'is_chunked' => true,
929		- 'chunk_count' => count( $chunks ),
930		- 'content_type' => $rows[0]->content_type,
931		- );
932		- } elseif ( $rows && count( $rows ) === 1 ) {
933		- $parsed = MxChat_Chunker::parse_stored_chunk( $rows[0]->article_content );
934		- return array(
935		- 'content' => $parsed['text'],
936		- 'source_url' => $source_url,
937		- 'entry_id' => $rows[0]->id,
938		- 'is_chunked' => false,
939		- 'content_type' => $rows[0]->content_type,
940		- );
941		- }
942		- }
943		-
944		- // Fallback: fetch by ID
945		- if ( $entry_id > 0 ) {
946		- $row = $wpdb->get_row( $wpdb->prepare(
947		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
948		- $entry_id
949		- ) );
950		- if ( $row ) {
951		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
952		- return array(
953		- 'content' => $parsed['text'],
954		- 'source_url' => $row->source_url,
955		- 'entry_id' => $row->id,
956		- 'is_chunked' => false,
957		- 'content_type' => $row->content_type,
958		- );
959		- }
960		- }
961		-
962		- return new WP_Error( 'not_found', 'Entry not found.' );
963		-}
964		-
965		-/**
966		- * Get content from Pinecone — fetches vectors by source_url, reassembles chunks.
967		- */
968		-private function get_pinecone_entry_content( $source_url, $entry_id, $bot_id ) {
969		- if ( ! class_exists('MxChat_Pinecone_Manager') ) {
970		- return new WP_Error( 'pinecone_unavailable', 'Pinecone manager not available.' );
971		- }
972		-
973		- // Get Pinecone config
974		- if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
975		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
976		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
977		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
978		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
979		- } else {
980		- $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
981		- $api_key = $bot_config['api_key'] ?? '';
982		- $host = $bot_config['host'] ?? '';
983		- $namespace = $bot_config['namespace'] ?? '';
984		- }
985		-
986		- if ( empty($host) || empty($api_key) ) {
987		- return new WP_Error( 'pinecone_config', 'Pinecone not configured.' );
988		- }
989		-
990		- // List vectors with the source_url prefix
991		- $base_id = md5( $source_url );
992		- $vector_ids = array( $base_id );
993		-
994		- // Find chunk vectors
995		- $list_url = "https://{$host}/vectors/list";
996		- $list_body = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
997		- if ( ! empty($namespace) ) {
998		- $list_body['namespace'] = $namespace;
999		- }
1000		-
1001		- $list_resp = wp_remote_post( $list_url, array(
1002		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1003		- 'body' => wp_json_encode( $list_body ),
1004		- 'timeout' => 15,
1005		- ) );
1006		-
1007		- if ( ! is_wp_error($list_resp) ) {
1008		- $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
1009		- if ( ! empty($list_data['vectors']) ) {
1010		- foreach ( $list_data['vectors'] as $v ) {
1011		- $vector_ids[] = $v['id'];
1012		- }
1013		- }
1014		- }
1015		-
1016		- // Fetch vectors with metadata
1017		- $fetch_url = "https://{$host}/vectors/fetch";
1018		- $fetch_body = array( 'ids' => $vector_ids );
1019		- if ( ! empty($namespace) ) {
1020		- $fetch_body['namespace'] = $namespace;
1021		- }
1022		-
1023		- $fetch_resp = wp_remote_post( $fetch_url, array(
1024		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1025		- 'body' => wp_json_encode( $fetch_body ),
1026		- 'timeout' => 15,
1027		- ) );
1028		-
1029		- if ( is_wp_error($fetch_resp) ) {
1030		- return new WP_Error( 'pinecone_fetch', 'Failed to fetch from Pinecone.' );
1031		- }
1032		-
1033		- $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
1034		- $vectors = $fetch_data['vectors'] ?? array();
1035		-
1036		- if ( empty($vectors) ) {
1037		- return new WP_Error( 'not_found', 'Entry not found in Pinecone.' );
1038		- }
1039		-
1040		- // Reassemble chunks
1041		- $chunks = array();
1042		- $content_type = 'content';
1043		- foreach ( $vectors as $vid => $vector ) {
1044		- $meta = $vector['metadata'] ?? array();
1045		- $text = $meta['text'] ?? '';
1046		- $index = $meta['chunk_index'] ?? 0;
1047		- $content_type = $meta['type'] ?? 'content';
1048		- $chunks[ intval($index) ] = $text;
1049		- }
1050		- ksort( $chunks );
1051		-
1052		- return array(
1053		- 'content' => implode( "\n\n", $chunks ),
1054		- 'source_url' => $source_url,
1055		- 'is_chunked' => count($chunks) > 1,
1056		- 'chunk_count' => count($chunks),
1057		- 'content_type' => $content_type,
1058		- );
1059		-}
1060		-
1061		-/**
1062		- * AJAX: Inspect a knowledge entry — returns the per-chunk stored text + metadata
1063		- * WITHOUT collapsing it, so a site owner can see exactly what was indexed for an
1064		- * entry (plan-mxchat-20260628-d8cb4b). READ-ONLY: never re-embeds or mutates.
1065		- */
1066		-public function ajax_mxchat_inspect_entry() {
1067		- check_ajax_referer('mxchat_inspect_entry_nonce', 'nonce');
1068		-
1069		- if ( ! current_user_can('manage_options') ) {
1070		- wp_send_json_error( array( 'message' => esc_html__('Permission denied.', 'mxchat') ) );
1071		- }
1072		-
1073		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
1074		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
1075		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
1076		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1077		-
1078		- if ( $data_source === 'pinecone' ) {
1079		- $result = $this->inspect_pinecone_entry( $source_url, $entry_id, $bot_id );
1080		- } else {
1081		- $result = $this->inspect_wordpress_entry( $source_url, $entry_id );
1082		- }
1083		-
1084		- if ( is_wp_error( $result ) ) {
1085		- wp_send_json_error( array( 'message' => $result->get_error_message() ) );
1086		- }
1087		-
1088		- wp_send_json_success( $result );
1089		-}
1090		-
1091		-/**
1092		- * Read-only inspector for WordPress-DB entries. Mirrors get_wordpress_entry_content()
1093		- * but returns each STORED chunk's exact text + length (no implode), plus the assembled
1094		- * embedded text. This shows what is actually in the index, not a re-derivation from the post.
1095		- */
1096		-private function inspect_wordpress_entry( $source_url, $entry_id ) {
1097		- global $wpdb;
1098		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
1099		-
1100		- $rows = array();
1101		-
1102		- // Group by the real stored source_url — this INCLUDES "mxchat://" manual
1103		- // Direct Content entries (the spec's manual-entry case), which share one
1104		- // source_url across their chunk rows. Only the synthetic "_ungrouped_<id>"
1105		- // display key (invented by the table view for rows with no source_url) is
1106		- // excluded; those fall through to the entry_id lookup below.
1107		- if ( ! empty( $source_url ) && strpos( $source_url, '_ungrouped_' ) !== 0 ) {
1108		- $rows = $wpdb->get_results( $wpdb->prepare(
1109		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
1110		- $source_url
1111		- ) );
1112		- }
1113		-
1114		- // Fallback / manual "Direct Content" entries: fetch the single row by id.
1115		- if ( empty( $rows ) && $entry_id > 0 ) {
1116		- $row = $wpdb->get_row( $wpdb->prepare(
1117		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
1118		- $entry_id
1119		- ) );
1120		- if ( $row ) {
1121		- $rows = array( $row );
1122		- }
1123		- }
1124		-
1125		- if ( empty( $rows ) ) {
1126		- return new WP_Error( 'not_found', esc_html__('Entry not found in the local knowledge database.', 'mxchat') );
1127		- }
1128		-
1129		- $chunks = array();
1130		- $content_type = '';
1131		- foreach ( $rows as $row ) {
1132		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
1133		- $text = isset( $parsed['text'] ) ? $parsed['text'] : '';
1134		- $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
1135		- $content_type = $row->content_type;
1136		- $chunks[] = array(
1137		- 'index' => $index,
1138		- 'text' => $text,
1139		- 'length' => function_exists('mb_strlen') ? mb_strlen( $text ) : strlen( $text ),
1140		- 'row_id' => intval( $row->id ),
1141		- );
1142		- }
1143		-
1144		- usort( $chunks, function( $a, $b ) { return $a['index'] - $b['index']; } );
1145		-
1146		- $assembled = implode( "\n\n", wp_list_pluck( $chunks, 'text' ) );
1147		-
1148		- return array(
1149		- 'store' => 'wordpress',
1150		- 'source_url' => $source_url,
1151		- 'content_type' => $content_type,
1152		- 'is_chunked' => count( $chunks ) > 1,
1153		- 'chunk_count' => count( $chunks ),
1154		- 'assembled' => $assembled,
1155		- 'assembled_length' => function_exists('mb_strlen') ? mb_strlen( $assembled ) : strlen( $assembled ),
1156		- 'chunks' => array_values( $chunks ),
1157		- // WP-DB storage carries no separate vector metadata; surface that fact
1158		- // rather than letting the owner guess (the spec's taxonomy question).
1159		- 'metadata' => array(),
1160		- 'metadata_note' => esc_html__('Stored in the local WordPress database. Only the assembled text shown here is embedded — there are no separate vector metadata fields (e.g. taxonomy terms are not stored unless they were injected into the text itself).', 'mxchat'),
1161		- );
1162		-}
1163		-
1164		-/**
1165		- * Read-only inspector for Pinecone entries. Mirrors get_pinecone_entry_content()
1166		- * but keeps each vector's text + metadata instead of imploding, so the owner can
1167		- * confirm exactly which metadata fields (text/source_url/type/last_updated/created_at/bot_id)
1168		- * are present per chunk. READ-ONLY.
1169		- */
1170		-private function inspect_pinecone_entry( $source_url, $entry_id, $bot_id ) {
1171		- if ( ! class_exists('MxChat_Pinecone_Manager') ) {
1172		- return new WP_Error( 'pinecone_unavailable', esc_html__('Pinecone manager not available.', 'mxchat') );
1173		- }
1174		-
1175		- if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
1176		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
1177		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1178		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1179		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1180		- } else {
1181		- $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1182		- $api_key = $bot_config['api_key'] ?? '';
1183		- $host = $bot_config['host'] ?? '';
1184		- $namespace = $bot_config['namespace'] ?? '';
1185		- }
1186		-
1187		- if ( empty($host) || empty($api_key) ) {
1188		- return new WP_Error( 'pinecone_config', esc_html__('Pinecone not configured.', 'mxchat') );
1189		- }
1190		-
1191		- $base_id = md5( $source_url );
1192		- $vector_ids = array( $base_id );
1193		-
1194		- $list_url = "https://{$host}/vectors/list";
1195		- $list_body = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
1196		- if ( ! empty($namespace) ) {
1197		- $list_body['namespace'] = $namespace;
1198		- }
1199		-
1200		- $list_resp = wp_remote_post( $list_url, array(
1201		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1202		- 'body' => wp_json_encode( $list_body ),
1203		- 'timeout' => 15,
1204		- ) );
1205		-
1206		- if ( ! is_wp_error($list_resp) ) {
1207		- $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
1208		- if ( ! empty($list_data['vectors']) ) {
1209		- foreach ( $list_data['vectors'] as $v ) {
1210		- $vector_ids[] = $v['id'];
1211		- }
1212		- }
1213		- }
1214		-
1215		- $fetch_url = "https://{$host}/vectors/fetch";
1216		- $fetch_body = array( 'ids' => $vector_ids );
1217		- if ( ! empty($namespace) ) {
1218		- $fetch_body['namespace'] = $namespace;
1219		- }
1220		-
1221		- $fetch_resp = wp_remote_post( $fetch_url, array(
1222		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1223		- 'body' => wp_json_encode( $fetch_body ),
1224		- 'timeout' => 15,
1225		- ) );
1226		-
1227		- if ( is_wp_error($fetch_resp) ) {
1228		- return new WP_Error( 'pinecone_fetch', esc_html__('Failed to fetch from Pinecone.', 'mxchat') );
1229		- }
1230		-
1231		- $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
1232		- $vectors = $fetch_data['vectors'] ?? array();
1233		-
1234		- if ( empty($vectors) ) {
1235		- return new WP_Error( 'not_found', esc_html__('Entry not found in Pinecone.', 'mxchat') );
1236		- }
1237		-
1238		- // Whitelisted metadata fields the spec calls out — shown so devs can confirm
1239		- // what is (and is NOT) stored per vector.
1240		- $meta_fields = array( 'text', 'source_url', 'type', 'last_updated', 'created_at', 'bot_id', 'chunk_index', 'total_chunks' );
1241		- $chunks = array();
1242		- $content_type = '';
1243		- foreach ( $vectors as $vid => $vector ) {
1244		- $meta = isset($vector['metadata']) && is_array($vector['metadata']) ? $vector['metadata'] : array();
1245		- $text = $meta['text'] ?? '';
1246		- $index = isset($meta['chunk_index']) ? intval($meta['chunk_index']) : count($chunks);
1247		- $content_type = $meta['type'] ?? $content_type;
1248		-
1249		- $clean_meta = array();
1250		- foreach ( $meta_fields as $field ) {
1251		- if ( array_key_exists( $field, $meta ) && $field !== 'text' ) {
1252		- $clean_meta[ $field ] = is_scalar( $meta[ $field ] ) ? (string) $meta[ $field ] : wp_json_encode( $meta[ $field ] );
1253		- }
1254		- }
1255		-
1256		- $chunks[] = array(
1257		- 'index' => $index,
1258		- 'text' => $text,
1259		- 'length' => function_exists('mb_strlen') ? mb_strlen( $text ) : strlen( $text ),
1260		- 'vector_id' => (string) $vid,
1261		- 'metadata' => $clean_meta,
1262		- );
1263		- }
1264		-
1265		- usort( $chunks, function( $a, $b ) { return $a['index'] - $b['index']; } );
1266		-
1267		- $assembled = implode( "\n\n", wp_list_pluck( $chunks, 'text' ) );
1268		-
1269		- return array(
1270		- 'store' => 'pinecone',
1271		- 'source_url' => $source_url,
1272		- 'content_type' => $content_type,
1273		- 'is_chunked' => count( $chunks ) > 1,
1274		- 'chunk_count' => count( $chunks ),
1275		- 'assembled' => $assembled,
1276		- 'assembled_length' => function_exists('mb_strlen') ? mb_strlen( $assembled ) : strlen( $assembled ),
1277		- 'chunks' => array_values( $chunks ),
1278		- 'metadata' => array(),
1279		- 'metadata_note' => esc_html__('Stored in Pinecone. Each chunk above lists the vector metadata fields actually present — if a field you expect (such as taxonomy terms) is missing here, it was not stored as metadata and is only searchable if it appears in the embedded text.', 'mxchat'),
1280		- );
1281		-}
1282		-
1283		-/**
1284		- * AJAX: Save edited content — re-chunks and re-embeds as needed.
1285		- * Works for both WordPress DB and Pinecone entries.
1286		- */
1287		-public function ajax_mxchat_save_entry_content() {
1288		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
1289		-
1290		- if ( ! current_user_can('manage_options') ) {
1291		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
1292		- }
1293		-
1294		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
1295		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
1296		- $content = isset($_POST['content']) ? wp_kses_post( wp_unslash($_POST['content']) ) : '';
1297		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
1298		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1299		- $content_type = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : 'content';
1300		-
1301		- if ( empty($content) ) {
1302		- wp_send_json_error( array( 'message' => 'Content cannot be empty.' ) );
1303		- }
1304		-
1305		- // Get the embedding API key
1306		- $options = get_option('mxchat_options', array());
1307		- $api_key = '';
1308		-
1309		- if ( $bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager') ) {
1310		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
1311		- $api_key = $bot_options['api_key'] ?? '';
1312		- }
1313		- if ( empty($api_key) ) {
1314		- $api_key = $options['api_key'] ?? '';
1315		- }
1316		-
1317		- global $wpdb;
1318		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
1319		-
1320		- // If source_url is empty but we have an entry_id, look it up
1321		- if ( empty($source_url) && $entry_id > 0 && $data_source === 'wordpress' ) {
1322		- $row = $wpdb->get_row( $wpdb->prepare( "SELECT source_url FROM {$table} WHERE id = %d", $entry_id ) );
1323		- if ( $row && ! empty($row->source_url) ) {
1324		- $source_url = $row->source_url;
1325		- }
1326		- }
1327		-
1328		- // For manual entries (no source_url or mxchat:// prefix), delete the old entry by ID first
1329		- // so submit_content_to_db creates a replacement instead of a duplicate
1330		- // Also treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
1331		- $is_legacy_manual = !empty($source_url) && strpos($source_url, 'mxchat.ai') !== false && strpos($source_url, 'mxchat://') !== 0;
1332		- if ( $entry_id > 0 && (empty($source_url) || strpos($source_url, 'mxchat://') === 0 || $is_legacy_manual) ) {
1333		- $wpdb->delete( $table, array( 'id' => $entry_id ), array( '%d' ) );
1334		- // Clear legacy URL so submit_content_to_db generates a unique mxchat:// identifier
1335		- // instead of reusing the shared URL (which would mass-delete other entries with the same URL)
1336		- if ( $is_legacy_manual ) {
1337		- $source_url = '';
1338		- }
1339		- }
1340		-
1341		- // Use the existing submit_content_to_db which handles chunking, Pinecone, and WP DB
1342		- $vector_id = ! empty($source_url) ? md5($source_url) : md5('mxchat_manual_' . $entry_id);
1343		-
1344		- // submit_content_to_db already handles: delete old chunks → re-chunk → re-embed → store
1345		- $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, $vector_id, $bot_id, $content_type );
1346		-
1347		- if ( is_wp_error($result) ) {
1348		- wp_send_json_error( array( 'message' => $result->get_error_message() ) );
1349		- }
1350		-
1351		- wp_send_json_success( array( 'message' => 'Content saved and re-embedded successfully.' ) );
1352		-}
1353		-
1354	775	public function mxchat_get_pdf_processing_status($pdf_url) {
1355	776	$pdf_url = esc_url_raw($pdf_url);
1356	777	$status = get_transient(sanitize_key('mxchat_pdf_status_' . md5($pdf_url)));
1357	778
		@@ -1394,18 +815,24 @@
1394	815	}
1395	816
1396	817
1397	818	public function mxchat_handle_sitemap_submission() {
	819	+ // Start logging the submission process
	820	+ //error_log('[MXCHAT-URL] ===== Starting URL submission process =====');
	821	+
1398	822	// Check if the form was submitted and verify permissions
1399	823	if (!isset($_POST['submit_sitemap']) || !current_user_can('manage_options')) {
	824	+ //error_log('[MXCHAT-URL] Error: Unauthorized access or form not submitted properly');
1400	825	wp_die(esc_html__('Unauthorized access', 'mxchat'));
1401	826	}
1402	827
1403	828	// Verify nonce
	829	+ //error_log('[MXCHAT-URL] Verifying nonce');
1404	830	check_admin_referer('mxchat_submit_sitemap_action', 'mxchat_submit_sitemap_nonce');
1405	831
1406	832	// Validate URL
1407	833	if (!isset($_POST['sitemap_url']) || empty($_POST['sitemap_url'])) {
	834	+ //error_log('[MXCHAT-URL] Error: Empty or missing URL');
1408	835	set_transient('mxchat_admin_notice_error',
1409	836	esc_html__('Please provide a valid URL.', 'mxchat'),
1410	837	30
1411	838	);
		@@ -1413,56 +840,43 @@
1413	840	exit;
1414	841	}
1415	842
1416	843	$submitted_url = esc_url_raw($_POST['sitemap_url']);
	844	+ //error_log('[MXCHAT-URL] Processing URL: ' . $submitted_url);
1417	845
1418		- // Convert Google Drive sharing URLs to direct download URLs
1419		- if ( strpos($submitted_url, 'drive.google.com') !== false ) {
1420		- $file_id = '';
1421		- if ( preg_match('/[?&]id=([a-zA-Z0-9_-]+)/', $submitted_url, $m) ) {
1422		- $file_id = $m[1];
1423		- } elseif ( preg_match('#/file/d/([a-zA-Z0-9_-]+)#', $submitted_url, $m) ) {
1424		- $file_id = $m[1];
1425		- }
1426		- if ( ! empty($file_id) ) {
1427		- $submitted_url = 'https://drive.google.com/uc?export=download&id=' . $file_id;
1428		- }
	846	+ // Validate API key first
	847	+ $options = get_option('mxchat_options');
	848	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	849	+
	850	+ if (strpos($selected_model, 'voyage') === 0) {
	851	+ $api_key = $options['voyage_api_key'] ?? '';
	852	+ $provider_name = 'Voyage AI';
	853	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	854	+ $api_key = $options['gemini_api_key'] ?? '';
	855	+ $provider_name = 'Google Gemini';
	856	+ } else {
	857	+ $api_key = $options['api_key'] ?? '';
	858	+ $provider_name = 'OpenAI';
1429	859	}
1430		-
1431		- // Get bot_id from form submission
1432		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1433		-
1434		- // Get bot-specific options and validate the embedding decision —
1435		- // custom-provider-aware (plan cbd5fd).
1436		- $bot_options = $this->get_bot_options($bot_id);
1437		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
1438		-
1439		- $preflight = MxChat_Utils::embedding_preflight($options);
1440		- if (!$preflight['ok']) {
1441		- set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
	860	+
	861	+ if (empty($api_key)) {
	862	+ $error_message = sprintf(
	863	+ esc_html__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
	864	+ $provider_name
	865	+ );
	866	+ //error_log('[MXCHAT-URL] Error: ' . $error_message);
	867	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	868	+ //error_log('[MXCHAT-URL] Set error transient: ' . $error_message);
1442	869	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1443	870	exit;
1444	871	}
1445		- $api_key = $preflight['api_key'];
1446	872
1447		- // Fetch URL — send an honest, versioned MXChat crawler UA (not a spoofed
1448		- // browser). Stale browser UAs are exactly what WAFs like SiteGround's
1449		- // ModSecurity flag as scrapers, 403-ing the fetch (including PDFs served
1450		- // from the site's own media library, which route through this same call).
1451		- // See mxchat_ingest_user_agent(). Accept is kept for content negotiation;
1452		- // the browser-only Accept-Language fingerprint is dropped so it stays
1453		- // coherent with a bot identity.
1454		- $response = wp_remote_get($submitted_url, array(
1455		- 'timeout' => 30,
1456		- 'sslverify' => false,
1457		- 'user-agent' => mxchat_ingest_user_agent(),
1458		- 'headers' => array(
1459		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1460		- ),
1461		- ));
	873	+ //error_log('[MXCHAT-URL] Fetching URL content');
	874	+ $response = wp_remote_get($submitted_url, array('timeout' => 30));
1462	875
1463	876	if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
1464	877	$error_message = is_wp_error($response) ? $response->get_error_message() : 'HTTP Status: ' . wp_remote_retrieve_response_code($response);
	878	+ //error_log('[MXCHAT-URL] Error fetching URL: ' . $error_message);
1465	879	set_transient('mxchat_admin_notice_error',
1466	880	sprintf(
1467	881	esc_html__('Failed to fetch the URL: %s', 'mxchat'),
1468	882	esc_html($error_message)
		@@ -1473,11 +887,13 @@
1473	887	exit;
1474	888	}
1475	889
1476	890	$content_type = wp_remote_retrieve_header($response, 'content-type');
	891	+ //error_log('[MXCHAT-URL] Content type: ' . $content_type);
1477	892	$body_content = wp_remote_retrieve_body($response);
1478	893
1479	894	if (empty($body_content)) {
	895	+ //error_log('[MXCHAT-URL] Error: Empty response body');
1480	896	set_transient('mxchat_admin_notice_error',
1481	897	esc_html__('Empty response received from URL.', 'mxchat'),
1482	898	30
1483	899	);
		@@ -1483,21 +899,29 @@
1483	899	);
1484	900	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1485	901	exit;
1486	902	}
	903	+ //error_log('[MXCHAT-URL] Retrieved body content length: ' . strlen($body_content) . ' bytes');
1487	904
1488	905	// Handle PDF URL
1489	906	if ($this->mxchat_is_pdf_url($submitted_url, $response)) {
1490		- $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response, $bot_id);
	907	+ //error_log('[MXCHAT-URL] Detected PDF URL, handling PDF for knowledge base');
	908	+ $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response);
	909	+ //error_log('[MXCHAT-URL] PDF handling result: ' . $result);
1491	910
1492		- if ($result === 'queued') {
1493		- set_transient('mxchat_admin_notice_success',
1494		- esc_html__('PDF queued for processing. Processing will start automatically.', 'mxchat'),
	911	+ if ($result === 'scheduled') {
	912	+ set_transient(
	913	+ 'mxchat_last_pdf_url',
	914	+ sanitize_text_field($submitted_url),
	915	+ DAY_IN_SECONDS
	916	+ );
	917	+ set_transient('mxchat_admin_notice_info',
	918	+ esc_html__('PDF processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1495	919	30
1496	920	);
1497	921	} else {
1498	922	set_transient('mxchat_admin_notice_error',
1499		- esc_html__('Failed to queue PDF processing: ', 'mxchat') . esc_html($result),
	923	+ esc_html__('Failed to start PDF processing: ', 'mxchat') . esc_html($result),
1500	924	30
1501	925	);
1502	926	}
1503	927
		@@ -1506,8 +930,9 @@
1506	930	}
1507	931
1508	932	// Handle Sitemap XML
1509	933	if (strpos($content_type, 'xml') !== false || strpos($body_content, '<urlset') !== false) {
	934	+ //error_log('[MXCHAT-URL] Detected XML content, processing as sitemap');
1510	935	libxml_use_internal_errors(true);
1511	936	$xml = simplexml_load_string($body_content);
1512	937	$xml_errors = libxml_get_errors();
1513	938	libxml_clear_errors();
		@@ -1512,8 +937,15 @@
1512	937	$xml_errors = libxml_get_errors();
1513	938	libxml_clear_errors();
1514	939
1515	940	if ($xml === false || !empty($xml_errors)) {
	941	+ //error_log('[MXCHAT-URL] Error: Invalid XML format');
	942	+ if (!empty($xml_errors)) {
	943	+ foreach ($xml_errors as $error) {
	944	+ //error_log('[MXCHAT-URL] XML Error: ' . $error->message);
	945	+ }
	946	+ }
	947	+
1516	948	set_transient('mxchat_admin_notice_error',
1517	949	esc_html__('Invalid sitemap XML. Please provide a valid sitemap.', 'mxchat'),
1518	950	30
1519	951	);
		@@ -1520,18 +952,27 @@
1520	952	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1521	953	exit;
1522	954	}
1523	955
1524		- $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url, $bot_id);
	956	+ //error_log('[MXCHAT-URL] Valid XML found, handling sitemap for knowledge base');
	957	+ $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url);
	958	+ //error_log('[MXCHAT-URL] Sitemap handling result: ' . $result);
1525	959
1526		- if ($result === 'queued') {
1527		- set_transient('mxchat_admin_notice_success',
1528		- esc_html__('Sitemap queued for processing. Processing will start automatically.', 'mxchat'),
	960	+ if ($result === 'scheduled') {
	961	+ set_transient(
	962	+ 'mxchat_last_sitemap_url',
	963	+ sanitize_text_field($submitted_url),
	964	+ DAY_IN_SECONDS
	965	+ );
	966	+ set_transient('mxchat_admin_notice_info',
	967	+ esc_html__('Sitemap processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1529	968	30
1530	969	);
1531	970	} else {
	971	+ // Return to the admin page without a redirect for better error display
	972	+ // The error is already stored in the sitemap status transient
1532	973	set_transient('mxchat_admin_notice_error',
1533		- esc_html__('Failed to queue sitemap processing. Please check the status below for details.', 'mxchat'),
	974	+ esc_html__('Failed to start sitemap processing. Please check the status below for details.', 'mxchat'),
1534	975	30
1535	976	);
1536	977	}
1537	978
		@@ -1538,50 +979,127 @@
1538	979	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1539	980	exit;
1540	981	}
1541	982
1542		- // Handle Regular URL (single page)
	983	+ // Handle Regular URL
	984	+ //error_log('[MXCHAT-URL] Processing as regular webpage');
1543	985	$page_content = $this->mxchat_extract_main_content($body_content);
	986	+ //error_log('[MXCHAT-URL] Extracted content length: ' . strlen($page_content) . ' bytes');
	987	+
1544	988	$sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	989	+ //error_log('[MXCHAT-URL] Sanitized content length: ' . strlen($sanitized_content) . ' bytes');
1545	990
1546		- //error_log('[MXCHAT-URL-DEBUG] URL: ' . $submitted_url);
1547		- //error_log('[MXCHAT-URL-DEBUG] Extracted page_content length: ' . strlen($page_content));
1548		- //error_log('[MXCHAT-URL-DEBUG] Sanitized content length: ' . strlen($sanitized_content));
1549		- //error_log('[MXCHAT-URL-DEBUG] Content preview: ' . substr($sanitized_content, 0, 500));
1550		-
1551	991	if (empty($sanitized_content)) {
	992	+ //error_log('[MXCHAT-URL] Error: No valid content after sanitization');
	993	+
	994	+ // Set both transients - the error notice and the URL status
1552	995	set_transient('mxchat_admin_notice_error',
1553	996	esc_html__('No valid content found on the provided URL.', 'mxchat'),
1554	997	30
1555	998	);
	999	+
	1000	+ // Set URL status transient
	1001	+ set_transient('mxchat_single_url_status', [
	1002	+ 'url' => $submitted_url,
	1003	+ 'timestamp' => current_time('mysql'),
	1004	+ 'status' => 'failed',
	1005	+ 'error' => esc_html__('No valid content found on the provided URL.', 'mxchat')
	1006	+ ], DAY_IN_SECONDS);
	1007	+
1556	1008	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1557	1009	exit;
1558	1010	}
1559	1011
1560		- // For single URLs, process immediately using submit_content_to_db
1561		- // This handles chunking automatically for large content
1562		- $db_result = MxChat_Utils::submit_content_to_db(
1563		- $sanitized_content,
1564		- $submitted_url,
1565		- $api_key,
1566		- null,
1567		- $bot_id,
1568		- 'url' // content_type
1569		- );
	1012	+ //error_log('[MXCHAT-URL] Generating embedding for content');
	1013	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
1570	1014
1571		- if (is_wp_error($db_result)) {
1572		- $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1015	+ // Check if embedding_vector is a string (error message)
	1016	+ if (is_string($embedding_vector)) {
	1017	+ //error_log('[MXCHAT-URL] Error generating embedding: ' . $embedding_vector);
	1018	+ $error_message = esc_html__('Failed to generate embedding: ', 'mxchat') . esc_html($embedding_vector);
	1019	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1020	+
	1021	+ // Set both transients
1573	1022	set_transient('mxchat_admin_notice_error', $error_message, 30);
1574		- } else {
	1023	+
	1024	+ // Set URL status transient
	1025	+ set_transient('mxchat_single_url_status', [
	1026	+ 'url' => $submitted_url,
	1027	+ 'timestamp' => current_time('mysql'),
	1028	+ 'status' => 'failed',
	1029	+ 'error' => $error_message
	1030	+ ], DAY_IN_SECONDS);
	1031	+
	1032	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1033	+ exit;
	1034	+ }
	1035	+
	1036	+ if (is_array($embedding_vector)) {
	1037	+ //error_log('[MXCHAT-URL] Successfully generated embedding with ' . count($embedding_vector) . ' dimensions');
	1038	+
	1039	+ $db_result = MxChat_Utils::submit_content_to_db(
	1040	+ $sanitized_content,
	1041	+ $submitted_url,
	1042	+ $api_key
	1043	+ );
	1044	+
	1045	+ if (is_wp_error($db_result)) {
	1046	+ //error_log('[MXCHAT-URL] Error: Failed to store content in database: ' . $db_result->get_error_message());
	1047	+ $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1048	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1049	+
	1050	+ // Set both transients
	1051	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1052	+
	1053	+ // Set URL status transient
	1054	+ set_transient('mxchat_single_url_status', [
	1055	+ 'url' => $submitted_url,
	1056	+ 'timestamp' => current_time('mysql'),
	1057	+ 'status' => 'failed',
	1058	+ 'error' => $error_message
	1059	+ ], DAY_IN_SECONDS);
	1060	+
	1061	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1062	+ exit;
	1063	+ }
	1064	+
	1065	+ //error_log('[MXCHAT-URL] Successfully stored content in database');
1575	1066	$success_message = esc_html__('URL content successfully submitted!', 'mxchat');
	1067	+ //error_log('[MXCHAT-URL] Setting success transient: ' . $success_message);
	1068	+
	1069	+ // Set both transients
1576	1070	set_transient('mxchat_admin_notice_success', $success_message, 30);
	1071	+
	1072	+ // Set URL status transient with success
	1073	+ set_transient('mxchat_single_url_status', [
	1074	+ 'url' => $submitted_url,
	1075	+ 'timestamp' => current_time('mysql'),
	1076	+ 'status' => 'complete',
	1077	+ 'content_length' => strlen($sanitized_content),
	1078	+ 'embedding_dimensions' => count($embedding_vector)
	1079	+ ], DAY_IN_SECONDS);
	1080	+
	1081	+ } else {
	1082	+ //error_log('[MXCHAT-URL] Error: Failed to generate embedding. Unexpected result type: ' . gettype($embedding_vector));
	1083	+ $error_message = esc_html__('Failed to generate embedding: Unexpected result type. Please check your API key and try again.', 'mxchat');
	1084	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1085	+
	1086	+ // Set both transients
	1087	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1088	+
	1089	+ // Set URL status transient
	1090	+ set_transient('mxchat_single_url_status', [
	1091	+ 'url' => $submitted_url,
	1092	+ 'timestamp' => current_time('mysql'),
	1093	+ 'status' => 'failed',
	1094	+ 'error' => $error_message
	1095	+ ], DAY_IN_SECONDS);
1577	1096	}
1578	1097
	1098	+ //error_log('[MXCHAT-URL] ===== Completed URL submission process =====');
1579	1099	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1580	1100	exit;
1581	1101	}
1582		-
1583		-
1584	1102	public function mxchat_get_single_url_status() {
1585	1103	$status = get_transient('mxchat_single_url_status');
1586	1104	if (!$status) {
1587	1105	return null;
		@@ -1593,11 +1111,13 @@
1593	1111	}
1594	1112
1595	1113	return $status;
1596	1114	}
1597		-
1598		-public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url, $bot_id = 'default') {
	1115	+public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url) {
	1116	+ // Clear any single URL status when starting sitemap processing
	1117	+ delete_transient('mxchat_single_url_status');
1599	1118	if (!current_user_can('manage_options')) {
	1119	+ //error_log(esc_html__('Unauthorized sitemap processing attempt', 'mxchat'));
1600	1120	return false;
1601	1121	}
1602	1122
1603	1123	try {
		@@ -1606,96 +1126,331 @@
1606	1126	if (!$xml || !is_object($xml)) {
1607	1127	throw new Exception(__('Invalid XML object provided', 'mxchat'));
1608	1128	}
1609	1129
1610		- // Get bot-specific embedding API for validation
1611		- $bot_options = $this->get_bot_options($bot_id);
1612		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	1130	+ // Add embedding validation before processing
	1131	+ // Test embedding with a small sample text to verify API key is working
	1132	+ $test_result = $this->mxchat_generate_embedding("This is a test to verify the embedding API key is working.");
1613	1133
1614		- // Test the embedding API before processing
1615		- $test_phrase = "Test embedding generation for MxChat";
1616		- $test_result = $this->mxchat_generate_embedding($test_phrase, $bot_id);
1617		-
	1134	+ // Check if test_result is a string (error message) rather than an array (valid embedding)
1618	1135	if (is_string($test_result)) {
	1136	+ //error_log('[MXCHAT-URL] Embedding API validation failed: ' . $test_result);
	1137	+
	1138	+ // Store the error in the status transient so it can be displayed later
	1139	+ $status_data = array(
	1140	+ 'total_urls' => 0,
	1141	+ 'processed_urls' => 0,
	1142	+ 'status' => 'error',
	1143	+ 'error' => __('Embedding API validation failed: ', 'mxchat') . $test_result,
	1144	+ 'last_update' => time()
	1145	+ );
	1146	+
	1147	+ set_transient(
	1148	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1149	+ array_map('sanitize_text_field', $status_data),
	1150	+ DAY_IN_SECONDS
	1151	+ );
	1152	+
1619	1153	throw new Exception(__('Embedding API validation failed: ', 'mxchat') . $test_result);
1620	1154	}
1621	1155
	1156	+ // Make sure it's an array (valid embedding)
1622	1157	if (!is_array($test_result)) {
	1158	+ //error_log('[MXCHAT-URL] Embedding API returned unexpected result type: ' . gettype($test_result));
1623	1159	throw new Exception(__('Embedding API returned unexpected result type. Please check your configuration.', 'mxchat'));
1624	1160	}
1625	1161
1626		- // Extract URLs from sitemap
1627		- $urls = array();
	1162	+ $urls = [];
1628	1163	foreach ($xml->url as $url_element) {
1629	1164	$url = esc_url_raw((string)$url_element->loc);
1630	1165	if ($url) {
1631		- $urls[] = array('url' => $url);
	1166	+ $urls[] = $url;
1632	1167	}
1633	1168	}
1634	1169
1635		- $total_urls = count($urls);
	1170	+ $total_urls = absint(count($urls));
1636	1171
1637	1172	if ($total_urls < 1) {
1638	1173	throw new Exception(__('No valid URLs found in sitemap', 'mxchat'));
1639	1174	}
1640	1175
1641		- // Create unique queue ID
1642		- $queue_id = 'sitemap_' . md5($sitemap_url . time());
	1176	+ wp_schedule_single_event(time(), 'mxchat_process_sitemap_urls', array(
	1177	+ 'urls' => $urls,
	1178	+ 'sitemap_url' => $sitemap_url,
	1179	+ 'total_urls' => $total_urls,
	1180	+ 'batch_size' => absint(10),
	1181	+ 'batch_pause' => absint(5)
	1182	+ ));
1643	1183
1644		- // Add URLs to queue
1645		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'url', $urls, $bot_id);
	1184	+ $status_data = array(
	1185	+ 'total_urls' => $total_urls,
	1186	+ 'processed_urls' => 0,
	1187	+ 'status' => 'processing',
	1188	+ 'last_update' => time()
	1189	+ );
1646	1190
1647		- if ($queued_count === 0) {
1648		- throw new Exception(__('Failed to add URLs to processing queue', 'mxchat'));
1649		- }
	1191	+ set_transient(
	1192	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1193	+ array_map('sanitize_text_field', $status_data),
	1194	+ DAY_IN_SECONDS
	1195	+ );
1650	1196
1651		- // Store queue metadata
1652		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $sitemap_url);
1653		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'sitemap');
1654		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_urls);
1655		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
1656		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	1197	+ return __('scheduled', 'mxchat');
1657	1198
1658		- // Store queue ID in transient for status tracking
1659		- set_transient('mxchat_active_queue_sitemap', $queue_id, DAY_IN_SECONDS);
1660		- set_transient('mxchat_last_sitemap_url', $sitemap_url, DAY_IN_SECONDS);
1661		-
1662		- return 'queued';
1663		-
1664		- } catch (Exception $e) {
	1199	+ } catch (\Exception $e) {
1665	1200	$error_message = $e->getMessage();
1666	1201	//error_log(sprintf(esc_html__('Error preparing sitemap for processing: %s', 'mxchat'), esc_html($error_message)));
1667	1202
	1203	+ // Store the sitemap URL and error in transients so they can be displayed
	1204	+ set_transient(
	1205	+ 'mxchat_last_sitemap_url',
	1206	+ sanitize_text_field($sitemap_url),
	1207	+ DAY_IN_SECONDS
	1208	+ );
	1209	+
	1210	+ $status_data = array(
	1211	+ 'total_urls' => 0,
	1212	+ 'processed_urls' => 0,
	1213	+ 'status' => 'error',
	1214	+ 'error' => $error_message,
	1215	+ 'last_update' => time()
	1216	+ );
	1217	+
	1218	+ set_transient(
	1219	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1220	+ array_map('sanitize_text_field', $status_data),
	1221	+ DAY_IN_SECONDS
	1222	+ );
	1223	+
1668	1224	return $error_message;
1669	1225	}
1670		-
1671	1226	}
1672	1227
1673		-/**
1674		- * Remove shortcode tags but preserve the content inside them
1675		- * Example: [vc_column]Hello World[/vc_column] becomes "Hello World"
1676		- *
1677		- * @param string $content The content containing shortcodes
1678		- * @return string Content with shortcode tags removed but inner content preserved
1679		- */
1680		-private function strip_shortcode_tags_preserve_content($content) {
1681		- // Single-pass regex removes all shortcode brackets: [tag], [tag attr="val"], [/tag], [tag /]
1682		- // Content between tags is inherently preserved since only brackets are targeted
1683		- $result = preg_replace('/\[\/?\w[\w-]*[^\]]*\]/', '', $content);
1684		- return ($result !== null) ? $result : $content;
	1228	+public function mxchat_process_sitemap_urls_cron($urls, $sitemap_url, $total_urls, $batch_size, $batch_pause) {
	1229	+ // Validate inputs
	1230	+ $sitemap_url = esc_url_raw($sitemap_url);
	1231	+ $total_urls = absint($total_urls);
	1232	+ $batch_size = absint($batch_size);
	1233	+ $batch_pause = absint($batch_pause);
	1234	+
	1235	+ if (!is_array($urls) || empty($urls)) {
	1236	+ return;
	1237	+ }
	1238	+
	1239	+ try {
	1240	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	1241	+ $status = get_transient($status_key);
	1242	+
	1243	+ if (!$status || !is_array($status)) {
	1244	+ throw new Exception('Invalid status data retrieved from transient');
	1245	+ }
	1246	+
	1247	+ // Initialize failed_urls array if it doesn't exist
	1248	+ if (!isset($status['failed_urls_list']) || !is_array($status['failed_urls_list'])) {
	1249	+ $status['failed_urls_list'] = [];
	1250	+ }
	1251	+
	1252	+ $start_url = absint($status['processed_urls']);
	1253	+ $end_url = min($start_url + $batch_size, $total_urls);
	1254	+
	1255	+ // Track batch statistics
	1256	+ $batch_stats = [
	1257	+ 'processed' => 0,
	1258	+ 'failed' => 0,
	1259	+ 'last_error' => '',
	1260	+ 'embedding_errors' => 0
	1261	+ ];
	1262	+
	1263	+ // Check embedding configuration with first URL (only on first batch)
	1264	+ if ($start_url === 0) {
	1265	+ $test_url = esc_url_raw($urls[0]);
	1266	+ $test_response = wp_remote_get($test_url);
	1267	+
	1268	+ if (!is_wp_error($test_response) && wp_remote_retrieve_response_code($test_response) === 200) {
	1269	+ $test_html = wp_remote_retrieve_body($test_response);
	1270	+ $test_content = $this->mxchat_extract_main_content($test_html);
	1271	+ $test_sanitized = $this->mxchat_sanitize_content_for_api($test_content);
	1272	+
	1273	+ if (!empty($test_sanitized)) {
	1274	+ $test_embedding = $this->mxchat_generate_embedding($test_sanitized);
	1275	+
	1276	+ if (is_string($test_embedding)) {
	1277	+ throw new Exception('Embedding generation failed: ' . $test_embedding);
	1278	+ }
	1279	+
	1280	+ if (!is_array($test_embedding)) {
	1281	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($test_embedding));
	1282	+ }
	1283	+ }
	1284	+ }
	1285	+ }
	1286	+
	1287	+ for ($i = $start_url; $i < $end_url; $i++) {
	1288	+ $page_url = esc_url_raw($urls[$i]);
	1289	+ $max_retries = 3;
	1290	+ $retry_count = 0;
	1291	+ $url_processed = false;
	1292	+ $last_error = '';
	1293	+
	1294	+ while (!$url_processed && $retry_count < $max_retries) {
	1295	+ try {
	1296	+ // Attempt to fetch the URL
	1297	+ $page_response = wp_remote_get($page_url, array('timeout' => 30));
	1298	+
	1299	+ if (is_wp_error($page_response)) {
	1300	+ throw new Exception('HTTP request failed: ' . $page_response->get_error_message());
	1301	+ }
	1302	+
	1303	+ $response_code = wp_remote_retrieve_response_code($page_response);
	1304	+ if ($response_code !== 200) {
	1305	+ throw new Exception('HTTP Status: ' . $response_code);
	1306	+ }
	1307	+
	1308	+ $page_html = wp_remote_retrieve_body($page_response);
	1309	+
	1310	+ if (empty($page_html)) {
	1311	+ throw new Exception('Empty response body');
	1312	+ }
	1313	+
	1314	+ $page_content = $this->mxchat_extract_main_content($page_html);
	1315	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	1316	+
	1317	+ if (empty($sanitized_content)) {
	1318	+ throw new Exception('No valid content found after processing');
	1319	+ }
	1320	+
	1321	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	1322	+
	1323	+ if (is_string($embedding_vector)) {
	1324	+ throw new Exception('Embedding generation failed: ' . $embedding_vector);
	1325	+ }
	1326	+
	1327	+ if (!is_array($embedding_vector)) {
	1328	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($embedding_vector));
	1329	+ }
	1330	+
	1331	+ // Submit to database
	1332	+ $options = get_option('mxchat_options');
	1333	+ $submission_result = MxChat_Utils::submit_content_to_db($sanitized_content, $page_url, $options['api_key']);
	1334	+
	1335	+ if (is_wp_error($submission_result)) {
	1336	+ throw new Exception('Database submission failed: ' . $submission_result->get_error_message());
	1337	+ }
	1338	+
	1339	+ // Success!
	1340	+ $url_processed = true;
	1341	+ $batch_stats['processed']++;
	1342	+
	1343	+ } catch (Exception $e) {
	1344	+ $retry_count++;
	1345	+ $last_error = $e->getMessage();
	1346	+
	1347	+ //error_log("URL {$page_url} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	1348	+
	1349	+ // Track embedding errors specifically
	1350	+ if (strpos($last_error, 'Embedding') !== false) {
	1351	+ $batch_stats['embedding_errors']++;
	1352	+ }
	1353	+
	1354	+ if ($retry_count < $max_retries) {
	1355	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	1356	+ sleep(pow(2, $retry_count - 1));
	1357	+ }
	1358	+ }
	1359	+ }
	1360	+
	1361	+ // If URL still not processed after all retries, mark as failed
	1362	+ if (!$url_processed) {
	1363	+ $batch_stats['failed']++;
	1364	+ $batch_stats['last_error'] = $last_error;
	1365	+
	1366	+ // Add to failed URLs list
	1367	+ $status['failed_urls_list'][] = [
	1368	+ 'url' => $page_url,
	1369	+ 'error' => $last_error,
	1370	+ 'time' => time(),
	1371	+ 'retries' => $max_retries
	1372	+ ];
	1373	+
	1374	+ // Limit the number of failed URLs we store to prevent transient size issues
	1375	+ if (count($status['failed_urls_list']) > 100) {
	1376	+ $status['failed_urls_list'] = array_slice($status['failed_urls_list'], -100);
	1377	+ }
	1378	+ }
	1379	+
	1380	+ // Update progress
	1381	+ $status['processed_urls'] = absint($i + 1);
	1382	+ $status['last_update'] = time();
	1383	+ $status['failed_urls'] = absint($status['failed_urls'] ?? 0) + ($url_processed ? 0 : 1);
	1384	+ $status['last_error'] = $batch_stats['last_error'];
	1385	+
	1386	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1387	+
	1388	+ // If we have too many consecutive embedding errors, stop processing
	1389	+ if ($batch_stats['embedding_errors'] >= 10) {
	1390	+ throw new Exception('Too many consecutive embedding failures detected. Please check your API configuration.');
	1391	+ }
	1392	+ }
	1393	+
	1394	+ // If all URLs in this batch failed, stop processing
	1395	+ if ($batch_stats['processed'] === 0 && $batch_stats['failed'] > 0) {
	1396	+ $status['status'] = 'error';
	1397	+ $status['error'] = sprintf(
	1398	+ 'Processing stopped: %d consecutive failures in batch. Last error: %s',
	1399	+ $batch_stats['failed'],
	1400	+ $batch_stats['last_error']
	1401	+ );
	1402	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1403	+ return;
	1404	+ }
	1405	+
	1406	+ // Update final progress
	1407	+ $status['processed_urls'] = min($end_url, $total_urls);
	1408	+ $status['last_update'] = time();
	1409	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1410	+
	1411	+ // Check if we've processed all URLs
	1412	+ if ($end_url >= $total_urls) {
	1413	+ // All URLs have been processed - mark as complete
	1414	+ $status['status'] = 'complete';
	1415	+ $status['processed_urls'] = $total_urls;
	1416	+
	1417	+ // Add completion summary
	1418	+ $status['completion_summary'] = [
	1419	+ 'total_urls' => $total_urls,
	1420	+ 'successful_urls' => $total_urls - absint($status['failed_urls'] ?? 0),
	1421	+ 'failed_urls' => absint($status['failed_urls'] ?? 0),
	1422	+ 'completion_time' => current_time('mysql')
	1423	+ ];
	1424	+
	1425	+ // Save the completed status (don't delete it - let user dismiss manually)
	1426	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1427	+
	1428	+ // DON'T delete the status transients here - let user dismiss manually
	1429	+ } else {
	1430	+ // Schedule next batch
	1431	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_sitemap_urls', array(
	1432	+ 'urls' => $urls,
	1433	+ 'sitemap_url' => $sitemap_url,
	1434	+ 'total_urls' => $total_urls,
	1435	+ 'batch_size' => $batch_size,
	1436	+ 'batch_pause' => $batch_pause,
	1437	+ ));
	1438	+ }
	1439	+ } catch (\Exception $e) {
	1440	+ $status['status'] = 'error';
	1441	+ $status['error'] = $e->getMessage();
	1442	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1443	+ }
1685	1444	}
1686		-
1687	1445	public function mxchat_sanitize_content_for_api($content) {
1688	1446	//error_log('[MXCHAT-SANITIZE] Original content preview: ' . substr($content, 0, 500) . '...');
1689		-
1690		- // Remove shortcode tags but PRESERVE content inside them
1691		- $content = $this->strip_shortcode_tags_preserve_content($content);
1692		-
	1447	+
1693	1448	// Remove script, style tags, and HTML comments
1694	1449	$content = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $content);
1695	1450	$content = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $content);
1696	1451	$content = preg_replace('/<!--(.|\s)*?-->/', '', $content);
1697		-
	1452	+
1698	1453	// Remove all HTML tags and decode HTML entities
1699	1454	$content = wp_strip_all_tags($content);
1700	1455	$content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5);
1701	1456
		@@ -1721,20 +1476,14 @@
1721	1476
1722	1477	// Ensure valid UTF-8 encoding
1723	1478	$content = wp_check_invalid_utf8($content);
1724	1479
1725		- // Remove extremely long runs with no whitespace (base64 blobs, minified JS).
1726		- // Counts CHARACTERS (/u), and never strips a run containing characters from a
1727		- // script written without spaces — Japanese, Chinese, Thai, Khmer, Lao, Myanmar —
1728		- // where a normal paragraph is legitimately one unbroken run.
1729		- $content = preg_replace_callback('/\S{300,}/u', function ($m) {
1730		- return preg_match('/[\p{Han}\p{Hiragana}\p{Katakana}\p{Thai}\p{Khmer}\p{Lao}\p{Myanmar}]/u', $m[0]) ? $m[0] : ' ';
1731		- }, $content);
1732		-
1733		- // Remove emoji/symbol blocks only — not the whole supplementary plane, which
1734		- // also holds CJK Extension B ideographs used in real Chinese/Japanese names
1735		- $content = preg_replace('/[\x{1F000}-\x{1F0FF}\x{1F300}-\x{1FAFF}]/u', '', $content);
	1480	+ // Remove any extremely long strings without spaces (often garbage)
	1481	+ $content = preg_replace('/\S{300,}/', ' ', $content);
1736	1482
	1483	+ // Replace problematic characters that often cause database issues
	1484	+ $content = preg_replace('/[\x{10000}-\x{10FFFF}]/u', '', $content); // Remove emoji and other high Unicode characters
	1485	+
1737	1486	// Replace any remaining potentially problematic characters with spaces
1738	1487	// BUT preserve newlines by temporarily replacing them
1739	1488	$content = str_replace("\n", "NEWLINE_PLACEHOLDER", $content);
1740	1489	$content = preg_replace('/[^\p{L}\p{N}\p{P}\p{Z}\p{Sm}]/u', ' ', $content);
		@@ -1739,13 +1488,12 @@
1739	1488	$content = str_replace("\n", "NEWLINE_PLACEHOLDER", $content);
1740	1489	$content = preg_replace('/[^\p{L}\p{N}\p{P}\p{Z}\p{Sm}]/u', ' ', $content);
1741	1490	$content = str_replace("NEWLINE_PLACEHOLDER", "\n", $content);
1742	1491
1743		- // Limit to reasonable length if needed (byte limit — MySQL TEXT is byte-sized,
1744		- // but cut on a character boundary so a multibyte char is never split mid-sequence)
	1492	+ // Limit to reasonable length if needed
1745	1493	$max_length = 65000; // Just under MySQL TEXT field limit
1746	1494	if (strlen($content) > $max_length) {
1747		- $content = mb_strcut($content, 0, $max_length, 'UTF-8');
	1495	+ $content = substr($content, 0, $max_length);
1748	1496	}
1749	1497
1750	1498	//error_log('[MXCHAT-SANITIZE] Sanitized content preview: ' . substr($content, 0, 500) . '...');
1751	1499	return $content;
		@@ -1760,12 +1508,12 @@
1760	1508	@$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'), LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
1761	1509	$xpath = new DOMXPath($dom);
1762	1510
1763	1511	// For debugging purposes
1764		- $debugEnabled = true; // Set to true to enable debugging output
	1512	+ $debugEnabled = false; // Set to true to enable debugging output
1765	1513	$debug = function($message) use ($debugEnabled) {
1766	1514	if ($debugEnabled) {
1767		- //error_log('[MXCHAT-EXTRACT-DEBUG] ' . $message);
	1515	+ //error_log('[MXCHAT-DEBUG] ' . $message);
1768	1516	}
1769	1517	};
1770	1518
1771	1519	// Direct targeting for Gerow theme posts
		@@ -1864,25 +1612,18 @@
1864	1612	'//main',
1865	1613	'//div[contains(@class, "content")]'
1866	1614	];
1867	1615
1868		- // First handle Elementor content - get only leaf widget containers to avoid duplicates
	1616	+ // First handle Elementor content
1869	1617	$debug("Checking for Elementor content");
1870		- // Get widget containers that are direct children of widgets (not nested inside other widget containers)
1871		- $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-widget")]//div[contains(@class, "elementor-widget-container")]');
	1618	+ $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-element")]//div[contains(@class, "elementor-widget-container")]');
1872	1619	if ($elementor_widgets && $elementor_widgets->length > 0) {
1873	1620	$debug("Found Elementor widgets");
1874		- $seen_content = array(); // Track seen content to avoid duplicates
1875	1621	$combined_content = '';
1876	1622	foreach ($elementor_widgets as $widget) {
1877	1623	$widget_content = $dom->saveHTML($widget);
1878	1624	if (!empty($widget_content)) {
1879		- // Create a hash of the content to detect duplicates
1880		- $content_hash = md5($widget_content);
1881		- if (!isset($seen_content[$content_hash])) {
1882		- $seen_content[$content_hash] = true;
1883		- $combined_content .= $widget_content;
1884		- }
	1625	+ $combined_content .= $widget_content;
1885	1626	}
1886	1627	}
1887	1628	if (!empty($combined_content)) {
1888	1629	$debug("Returning Elementor content");
		@@ -1894,14 +1635,15 @@
1894	1635	foreach ($selectors as $selector) {
1895	1636	$debug("Trying selector: " . $selector);
1896	1637	$nodes = $xpath->query($selector);
1897	1638	if ($nodes && $nodes->length > 0) {
1898		- $debug("Found " . $nodes->length . " matches for selector: " . $selector);
1899		- // Only take the FIRST matching node to avoid duplicate content
1900		- // (pages often have nested or multiple containers with same class)
1901		- $content = $dom->saveHTML($nodes->item(0));
	1639	+ $debug("Found matches for selector: " . $selector);
	1640	+ $content = '';
	1641	+ foreach ($nodes as $node) {
	1642	+ $content .= $dom->saveHTML($node);
	1643	+ }
1902	1644	if (!empty($content)) {
1903		- $debug("Returning content from selector: " . $selector . " (first match only)");
	1645	+ $debug("Returning content from selector: " . $selector);
1904	1646	return $content;
1905	1647	}
1906	1648	}
1907	1649	}
		@@ -1925,106 +1667,16 @@
1925	1667	$debug("Returning blog-area section content");
1926	1668	return $content;
1927	1669	}
1928	1670	}
1929		-
1930		- // Generic container selectors for non-CMS sites (like .asp pages)
1931		- $debug("Trying generic container selectors");
1932		- $generic_selectors = [
1933		- '//div[@id="main"]',
1934		- '//div[@id="wrapper"]',
1935		- '//div[@id="page"]',
1936		- '//div[@id="site-content"]',
1937		- '//div[contains(@class, "main-content")]',
1938		- '//div[contains(@class, "page-content")]',
1939		- '//div[contains(@class, "site-content")]',
1940		- ];
1941		-
1942		- foreach ($generic_selectors as $selector) {
1943		- $debug("Trying generic selector: " . $selector);
1944		- $nodes = $xpath->query($selector);
1945		- if ($nodes && $nodes->length > 0) {
1946		- $content = $dom->saveHTML($nodes->item(0));
1947		- if (!empty($content)) {
1948		- $debug("Returning content from generic selector: " . $selector);
1949		- return $content;
1950		- }
1951		- }
1952		- }
1953		-
1954		- // Paragraph-based content detection - find regions with substantial text
1955		- $debug("Trying paragraph-based content detection");
1956		- $paragraphs = $xpath->query('//p[string-length(normalize-space()) > 50]');
1957		- if ($paragraphs && $paragraphs->length >= 3) {
1958		- $debug("Found " . $paragraphs->length . " substantial paragraphs");
1959		- // Collect all substantial paragraphs and their content
1960		- $paragraph_content = '';
1961		- foreach ($paragraphs as $p) {
1962		- $paragraph_content .= $dom->saveHTML($p) . "\n";
1963		- }
1964		- if (!empty($paragraph_content)) {
1965		- $debug("Returning paragraph-based content");
1966		- return $paragraph_content;
1967		- }
1968		- }
1969		-
1970		- // Improved body fallback - strip nav/header/footer elements first
1971		- $debug("Using improved body fallback");
	1671	+
	1672	+ // Fallback: Return the body content if no specific selector matches
	1673	+ $debug("Using body fallback");
1972	1674	$body = $dom->getElementsByTagName('body');
1973	1675	if ($body->length > 0) {
1974		- // Clone the body to avoid modifying the original DOM
1975		- $body_clone = $body->item(0)->cloneNode(true);
1976		-
1977		- // Remove common non-content elements by tag name
1978		- $remove_tags = ['nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript'];
1979		- foreach ($remove_tags as $tag) {
1980		- $elements = $body_clone->getElementsByTagName($tag);
1981		- // Iterate backwards to safely remove elements
1982		- for ($i = $elements->length - 1; $i >= 0; $i--) {
1983		- $el = $elements->item($i);
1984		- if ($el && $el->parentNode) {
1985		- $el->parentNode->removeChild($el);
1986		- }
1987		- }
1988		- }
1989		-
1990		- // Remove elements with common non-content class names using XPath on the cloned body
1991		- $temp_dom = new DOMDocument();
1992		- @$temp_dom->appendChild($temp_dom->importNode($body_clone, true));
1993		- $temp_xpath = new DOMXPath($temp_dom);
1994		-
1995		- $remove_class_patterns = [
1996		- '//*[contains(@class, "nav")]',
1997		- '//*[contains(@class, "menu")]',
1998		- '//*[contains(@class, "sidebar")]',
1999		- '//*[contains(@class, "footer")]',
2000		- '//*[contains(@class, "header")]',
2001		- '//*[contains(@id, "nav")]',
2002		- '//*[contains(@id, "menu")]',
2003		- '//*[contains(@id, "sidebar")]',
2004		- '//*[contains(@id, "footer")]',
2005		- '//*[contains(@id, "header")]',
2006		- ];
2007		-
2008		- foreach ($remove_class_patterns as $pattern) {
2009		- $elements = $temp_xpath->query($pattern);
2010		- if ($elements) {
2011		- for ($i = $elements->length - 1; $i >= 0; $i--) {
2012		- $el = $elements->item($i);
2013		- if ($el && $el->parentNode) {
2014		- $el->parentNode->removeChild($el);
2015		- }
2016		- }
2017		- }
2018		- }
2019		-
2020		- $cleaned_content = $temp_dom->saveHTML();
2021		- if (!empty($cleaned_content)) {
2022		- $debug("Returning cleaned body content");
2023		- return $cleaned_content;
2024		- }
	1676	+ return $dom->saveHTML($body->item(0));
2025	1677	}
2026		-
	1678	+
2027	1679	// Last resort: return the original HTML
2028	1680	$debug("Returning original HTML");
2029	1681	return $html;
2030	1682	} catch (Exception $e) {
		@@ -2076,41 +1728,47 @@
2076	1728	try {
2077	1729	// Verify the request
2078	1730	check_ajax_referer('mxchat_status_nonce', 'nonce');
2079	1731
2080		- // Get active queue IDs
2081		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
2082		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
	1732	+ // Get the status just like in your admin page
	1733	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1734	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
2083	1735
2084		- $sitemap_status = false;
2085		- $pdf_status = false;
	1736	+ $pdf_status = $pdf_url ? $this->mxchat_get_pdf_processing_status($pdf_url) : false;
	1737	+ $sitemap_status = $sitemap_url ? $this->mxchat_get_sitemap_processing_status($sitemap_url) : false;
2086	1738
2087		- // Get sitemap queue status
2088		- if ($sitemap_queue_id) {
2089		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
	1739	+ // Add the PDF URL to the status object
	1740	+ if ($pdf_status && $pdf_url) {
	1741	+ $pdf_status['pdf_url'] = $pdf_url;
2090	1742	}
2091	1743
2092		- // Get PDF queue status
2093		- if ($pdf_queue_id) {
2094		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
2095		- }
	1744	+ // Set the current PDF URL for the manual batch processing button
	1745	+ $current_pdf_url = $pdf_url;
2096	1746
	1747	+ // Check for true processing status, not just presence of status
2097	1748	$is_active_processing =
2098		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
2099		- ($pdf_status && $pdf_status['status'] === 'processing');
	1749	+ ($sitemap_status && isset($sitemap_status['status']) && $sitemap_status['status'] === 'processing') ||
	1750	+ ($pdf_status && isset($pdf_status['status']) && $pdf_status['status'] === 'processing');
2100	1751
	1752	+ // Get single URL status, but only if no processing is active
	1753	+ $single_url_status = !$is_active_processing ? $this->mxchat_get_single_url_status() : false;
	1754	+
	1755	+ // REMOVED: Auto-clearing of completed status - now only done via dismiss button
	1756	+
2101	1757	// Return JSON response with the status data
2102	1758	wp_send_json(array(
2103	1759	'pdf_status' => $pdf_status,
2104	1760	'sitemap_status' => $sitemap_status,
	1761	+ 'single_url_status' => $single_url_status,
2105	1762	'is_processing' => $is_active_processing,
2106		- 'sitemap_queue_id' => $sitemap_queue_id,
2107		- 'pdf_queue_id' => $pdf_queue_id
	1763	+ 'current_pdf_url' => $current_pdf_url
2108	1764	));
2109	1765
2110	1766	} catch (Exception $e) {
	1767	+ // Log the error
2111	1768	//error_log('MxChat Status Update Error: ' . $e->getMessage());
2112	1769
	1770	+ // Return a friendly error response
2113	1771	wp_send_json_error(array(
2114	1772	'message' => 'Error getting status updates: ' . $e->getMessage(),
2115	1773	'status' => 'error'
2116	1774	));
		@@ -2115,1371 +1773,8 @@
2115	1773	'status' => 'error'
2116	1774	));
2117	1775	}
2118	1776	}
2119		-
2120		-/**
2121		- * Helper function to get queue status data
2122		- */
2123		-private function mxchat_get_queue_status_data($queue_id, $type = 'sitemap') {
2124		- global $wpdb;
2125		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
2126		-
2127		- // Get counts by status
2128		- $counts = $wpdb->get_results($wpdb->prepare(
2129		- "SELECT status, COUNT(*) as count
2130		- FROM $table_name
2131		- WHERE queue_id = %s
2132		- GROUP BY status",
2133		- $queue_id
2134		- ), OBJECT_K);
2135		-
2136		- $total = 0;
2137		- $completed = 0;
2138		- $failed = 0;
2139		- $processing = 0;
2140		- $pending = 0;
2141		-
2142		- foreach ($counts as $status => $data) {
2143		- $count = absint($data->count);
2144		- $total += $count;
2145		-
2146		- switch ($status) {
2147		- case 'completed':
2148		- $completed = $count;
2149		- break;
2150		- case 'failed':
2151		- $failed = $count;
2152		- break;
2153		- case 'processing':
2154		- $processing = $count;
2155		- break;
2156		- case 'pending':
2157		- $pending = $count;
2158		- break;
2159		- }
2160		- }
2161		-
2162		- if ($total === 0) {
2163		- return false;
2164		- }
2165		-
2166		- // Calculate percentage
2167		- $percentage = round((($completed + $failed) / $total) * 100);
2168		-
2169		- // Get failed items details (limit to 50)
2170		- $failed_items = array();
2171		- if ($failed > 0) {
2172		- $failed_results = $wpdb->get_results($wpdb->prepare(
2173		- "SELECT item_type, item_data, error_message, attempts, completed_at
2174		- FROM $table_name
2175		- WHERE queue_id = %s
2176		- AND status = 'failed'
2177		- AND attempts >= max_attempts
2178		- ORDER BY id DESC
2179		- LIMIT 50",
2180		- $queue_id
2181		- ));
2182		-
2183		- foreach ($failed_results as $item) {
2184		- $data = json_decode($item->item_data, true);
2185		- $url = $type === 'pdf' ? ($data['pdf_url'] ?? '') : ($data['url'] ?? '');
2186		-
2187		- $failed_items[] = array(
2188		- 'url' => $url,
2189		- 'page' => $type === 'pdf' ? ($data['page_number'] ?? 0) : 0,
2190		- 'error' => $item->error_message,
2191		- 'retries' => $item->attempts,
2192		- 'time' => strtotime($item->completed_at)
2193		- );
2194		- }
2195		- }
2196		-
2197		- // Get queue metadata
2198		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
2199		-
2200		- // Determine if queue is complete
2201		- $is_complete = ($pending === 0 && $processing === 0);
2202		-
2203		- // Get last update time
2204		- $last_update = $wpdb->get_var($wpdb->prepare(
2205		- "SELECT MAX(UNIX_TIMESTAMP(COALESCE(completed_at, started_at, created_at)))
2206		- FROM $table_name
2207		- WHERE queue_id = %s",
2208		- $queue_id
2209		- ));
2210		-
2211		- $last_update_text = $last_update ? human_time_diff($last_update, time()) . ' ' . __('ago', 'mxchat') : __('Just now', 'mxchat');
2212		-
2213		- // Format based on type
2214		- if ($type === 'pdf') {
2215		- return array(
2216		- 'total_pages' => $total,
2217		- 'processed_pages' => $completed + $failed,
2218		- 'failed_pages' => $failed,
2219		- 'percentage' => $percentage,
2220		- 'status' => $is_complete ? 'complete' : 'processing',
2221		- 'last_update' => $last_update_text,
2222		- 'failed_pages_list' => $failed_items,
2223		- 'pdf_url' => $source_url,
2224		- 'queue_id' => $queue_id
2225		- );
2226		- } else {
2227		- return array(
2228		- 'total_urls' => $total,
2229		- 'processed_urls' => $completed + $failed,
2230		- 'failed_urls' => $failed,
2231		- 'percentage' => $percentage,
2232		- 'status' => $is_complete ? 'complete' : 'processing',
2233		- 'last_update' => $last_update_text,
2234		- 'failed_urls_list' => $failed_items,
2235		- 'sitemap_url' => $source_url,
2236		- 'queue_id' => $queue_id
2237		- );
2238		- }
2239		-}
2240		-
2241		-/**
2242		- * Public method to get processing status for both sitemap and PDF queues
2243		- * Used by admin pages to display processing status
2244		- *
2245		- * @return array Array with 'sitemap_status', 'pdf_status', and 'is_processing' keys
2246		- */
2247		-public function mxchat_get_processing_statuses() {
2248		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
2249		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
2250		-
2251		- $sitemap_status = false;
2252		- $pdf_status = false;
2253		-
2254		- if ($sitemap_queue_id) {
2255		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
2256		- }
2257		-
2258		- if ($pdf_queue_id) {
2259		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
2260		- }
2261		-
2262		- $is_processing =
2263		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
2264		- ($pdf_status && $pdf_status['status'] === 'processing');
2265		-
2266		- return array(
2267		- 'sitemap_status' => $sitemap_status,
2268		- 'pdf_status' => $pdf_status,
2269		- 'is_processing' => $is_processing
2270		- );
2271		-}
2272		-
2273		-/**
2274		- * AJAX handler to get recent knowledge entries for real-time table updates
2275		- * UPDATED: Now supports both WordPress DB and Pinecone data sources
2276		- */
2277		-public function ajax_mxchat_get_recent_entries() {
2278		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2279		-
2280		- if (!current_user_can('manage_options')) {
2281		- wp_send_json_error(array('message' => 'Unauthorized'));
2282		- return;
2283		- }
2284		-
2285		- global $wpdb;
2286		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2287		-
2288		- // Get parameters
2289		- $last_id = isset($_POST['last_id']) ? absint($_POST['last_id']) : 0;
2290		- $limit = isset($_POST['limit']) ? min(absint($_POST['limit']), 50) : 10;
2291		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2292		-
2293		- // Check if Pinecone is enabled for this bot
2294		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
2295		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
2296		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2297		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
2298		-
2299		- if ($use_pinecone && $has_pinecone_api) {
2300		- // PINECONE DATA SOURCE - Get grouped entry count (not raw vector count)
2301		- // Use mxchat_fetch_pinecone_records which returns total_unique_entries
2302		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, '', 1, 10, $bot_id, '');
2303		- $total_count = $records['total'] ?? 0;
2304		-
2305		- // For Pinecone, we don't return individual entries during polling
2306		- // (entries are already displayed on page load via mxchat_fetch_pinecone_records)
2307		- // We just return the updated count
2308		- wp_send_json_success(array(
2309		- 'entries' => array(),
2310		- 'total_count' => absint($total_count),
2311		- 'max_id' => $last_id,
2312		- 'data_source' => 'pinecone'
2313		- ));
2314		- return;
2315		- }
2316		-
2317		- // WORDPRESS DB DATA SOURCE
2318		- // Build query to get entries newer than last_id
2319		- $where_clauses = array('1=1');
2320		- $where_values = array();
2321		-
2322		- if ($last_id > 0) {
2323		- $where_clauses[] = 'id > %d';
2324		- $where_values[] = $last_id;
2325		- }
2326		-
2327		- // Note: WordPress DB table doesn't have bot_id column
2328		- // Multi-bot filtering is handled via Pinecone namespaces
2329		-
2330		- $where_sql = implode(' AND ', $where_clauses);
2331		-
2332		- // Get recent entries
2333		- $query = "SELECT id, article_content, source_url, timestamp
2334		- FROM $table_name
2335		- WHERE $where_sql
2336		- ORDER BY id DESC
2337		- LIMIT %d";
2338		-
2339		- $where_values[] = $limit;
2340		-
2341		- $entries = $wpdb->get_results($wpdb->prepare($query, $where_values));
2342		-
2343		- // Get total count of GROUPED entries (by source_url) - matches pagination display
2344		- // Count unique source_urls (excluding mxchat:// internal refs) + count of ungrouped rows
2345		- $total_count = $wpdb->get_var(
2346		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2347		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
2348		- );
2349		-
2350		- // Format entries for response
2351		- $formatted_entries = array();
2352		- $preview_length = 150;
2353		- foreach ($entries as $entry) {
2354		- // Parse chunk metadata using the proper chunker method (same as initial page load)
2355		- if (class_exists('MxChat_Chunker')) {
2356		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($entry->article_content);
2357		- $display_content = $chunk_meta['text'];
2358		- $chunk_metadata = $chunk_meta['metadata'];
2359		- } else {
2360		- $display_content = $entry->article_content;
2361		- $chunk_metadata = array();
2362		- }
2363		-
2364		- $content_preview = mb_strlen($display_content) > $preview_length
2365		- ? mb_substr($display_content, 0, $preview_length) . '...'
2366		- : $display_content;
2367		-
2368		- $formatted_entries[] = array(
2369		- 'id' => $entry->id,
2370		- 'preview' => esc_html($content_preview),
2371		- 'full_content' => wp_kses_post(wpautop($display_content)),
2372		- 'content_length' => mb_strlen($display_content),
2373		- 'preview_length' => $preview_length,
2374		- 'source_url' => $entry->source_url,
2375		- 'has_link' => !empty($entry->source_url) && strpos($entry->source_url, 'mxchat://') !== 0,
2376		- 'chunk_metadata' => $chunk_metadata,
2377		- 'bot_id' => $entry->bot_id ?? 'default',
2378		- 'edit_nonce' => wp_create_nonce('mxchat_edit_entry_nonce'),
2379		- 'delete_nonce' => wp_create_nonce('mxchat_delete_prompt_nonce')
2380		- );
2381		- }
2382		-
2383		- wp_send_json_success(array(
2384		- 'entries' => $formatted_entries,
2385		- 'total_count' => absint($total_count),
2386		- 'max_id' => !empty($entries) ? $entries[0]->id : $last_id,
2387		- 'data_source' => 'wordpress'
2388		- ));
2389		-}
2390		-
2391		-/**
2392		- * Get Pinecone total count from stats API
2393		- * Helper function for ajax_mxchat_get_recent_entries
2394		- */
2395		-private function mxchat_get_pinecone_count_from_stats($pinecone_options) {
2396		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2397		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
2398		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
2399		-
2400		- if (empty($api_key) || empty($host)) {
2401		- return 0;
2402		- }
2403		-
2404		- try {
2405		- $stats_url = "https://{$host}/describe_index_stats";
2406		-
2407		- $response = wp_remote_post($stats_url, array(
2408		- 'headers' => array(
2409		- 'Api-Key' => $api_key,
2410		- 'Content-Type' => 'application/json'
2411		- ),
2412		- 'body' => '{}',
2413		- 'timeout' => 10
2414		- ));
2415		-
2416		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2417		- $body = wp_remote_retrieve_body($response);
2418		- $stats_data = json_decode($body, true);
2419		-
2420		- // If namespace is specified, get count from that specific namespace
2421		- if (!empty($namespace) && isset($stats_data['namespaces'][$namespace]['vectorCount'])) {
2422		- return intval($stats_data['namespaces'][$namespace]['vectorCount']);
2423		- }
2424		-
2425		- // If no namespace specified or namespace not found in response, use total
2426		- return intval($stats_data['totalVectorCount'] ?? 0);
2427		- }
2428		-
2429		- return 0;
2430		-
2431		- } catch (Exception $e) {
2432		- return 0;
2433		- }
2434		-}
2435		-
2436		-/**
2437		- * AJAX handler to refresh Pinecone entries table via AJAX
2438		- * Returns the table HTML for updating the UI without a full page reload
2439		- */
2440		-public function ajax_mxchat_refresh_pinecone_entries() {
2441		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2442		-
2443		- if (!current_user_can('manage_options')) {
2444		- wp_send_json_error(array('message' => 'Unauthorized'));
2445		- return;
2446		- }
2447		-
2448		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2449		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2450		- $per_page = 25;
2451		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2452		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2453		-
2454		- // Get Pinecone manager and options
2455		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
2456		- if (!$pinecone_manager) {
2457		- wp_send_json_error(array('message' => 'Pinecone manager not available'));
2458		- return;
2459		- }
2460		-
2461		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
2462		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2463		- $pinecone_api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2464		-
2465		- if (!$use_pinecone || empty($pinecone_api_key)) {
2466		- wp_send_json_error(array('message' => 'Pinecone not configured'));
2467		- return;
2468		- }
2469		-
2470		- // Fetch records from Pinecone
2471		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, $search_query, $page, $per_page, $bot_id, $content_type_filter);
2472		- $prompts = $records['data'] ?? array();
2473		- $total_records = $records['total'] ?? 0;
2474		-
2475		- // Preprocess Pinecone records — set chunk_metadata and display_content
2476		- // (matches admin-knowledge-page.php preprocessing)
2477		- foreach ($prompts as $prompt) {
2478		- if (isset($prompt->chunk_index) && $prompt->chunk_index !== null) {
2479		- $prompt->chunk_metadata = array(
2480		- 'chunk_index' => intval($prompt->chunk_index),
2481		- 'total_chunks' => isset($prompt->total_chunks) ? intval($prompt->total_chunks) : null,
2482		- 'is_chunked' => isset($prompt->is_chunked) ? (bool) $prompt->is_chunked : true,
2483		- 'source_url' => $prompt->source_url ?? ''
2484		- );
2485		- $prompt->display_content = $prompt->article_content;
2486		- } else {
2487		- $prompt->chunk_metadata = array();
2488		- $prompt->display_content = $prompt->article_content ?? '';
2489		- }
2490		- }
2491		-
2492		- // Group prompts by source_url
2493		- $grouped_prompts = array();
2494		- foreach ($prompts as $prompt) {
2495		- $source_url = '';
2496		- if (!empty($prompt->chunk_metadata['source_url'])) {
2497		- $source_url = $prompt->chunk_metadata['source_url'];
2498		- } elseif (!empty($prompt->source_url)) {
2499		- $source_url = $prompt->source_url;
2500		- }
2501		-
2502		- if (!empty($source_url)) {
2503		- if (!isset($grouped_prompts[$source_url])) {
2504		- $grouped_prompts[$source_url] = array();
2505		- }
2506		- $grouped_prompts[$source_url][] = $prompt;
2507		- } else {
2508		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2509		- }
2510		- }
2511		-
2512		- // Sort each group by chunk_index
2513		- foreach ($grouped_prompts as $source_url => &$group) {
2514		- usort($group, function($a, $b) {
2515		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2516		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2517		- return $index_a - $index_b;
2518		- });
2519		- }
2520		- unset($group);
2521		-
2522		- // Build HTML for the table rows - must match admin-knowledge-page.php structure exactly
2523		- ob_start();
2524		- $display_index = 0;
2525		- $current_page = $page;
2526		- $data_source = 'pinecone';
2527		- $current_bot_id = $bot_id;
2528		- $preview_length = 150;
2529		-
2530		- if (empty($grouped_prompts)) {
2531		- echo '<tr><td colspan="4" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2532		- esc_html_e('No knowledge entries found in Pinecone.', 'mxchat');
2533		- echo '</td></tr>';
2534		- } else {
2535		- foreach ($grouped_prompts as $source_url => $group) {
2536		- $chunk_count = count($group);
2537		- $first_prompt = $group[0];
2538		- $display_index++;
2539		-
2540		- if ($chunk_count > 1) {
2541		- // Multiple chunks - show grouped row with expand button
2542		- $group_id = 'group-' . md5($source_url);
2543		- ?>
2544		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2545		- class="mxchat-chunk-group-header"
2546		- data-source="<?php echo esc_attr($data_source); ?>"
2547		- data-group-id="<?php echo esc_attr($group_id); ?>"
2548		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2549		- <td style="padding: 12px 16px; text-align: center;">
2550		- <input type="checkbox"
2551		- class="mxchat-entry-checkbox"
2552		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2553		- data-source="<?php echo esc_attr($data_source); ?>"
2554		- data-source-url="<?php echo esc_attr($source_url); ?>"
2555		- data-is-group="true"
2556		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2557		- </td>
2558		- <td style="padding: 12px 16px; font-size: 13px;">
2559		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2560		- </td>
2561		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2562		- <div class="mxchat-chunk-group-info">
2563		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2564		- <span class="dashicons dashicons-arrow-right-alt2"></span>
2565		- </button>
2566		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2567		- <span class="mxchat-chunk-preview">
2568		- <?php
2569		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : (isset($first_prompt->article_content) ? $first_prompt->article_content : '');
2570		- $content_preview = mb_substr($parent_content, 0, 100);
2571		- echo esc_html($content_preview . '...');
2572		- ?>
2573		- </span>
2574		- </div>
2575		- </td>
2576		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2577		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2578		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2579		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2580		- <?php esc_html_e('View Source', 'mxchat'); ?>
2581		- </a>
2582		- <?php else : ?>
2583		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2584		- <?php endif; ?>
2585		- </td>
2586		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2587		- <?php if ($data_source !== 'pinecone') : ?>
2588		- <button type="button"
2589		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2590		- data-source-url="<?php echo esc_attr($source_url); ?>"
2591		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2592		- data-data-source="<?php echo esc_attr($data_source); ?>"
2593		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2594		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2595		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2596		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2597		- </button>
2598		- <?php endif; ?>
2599		- <button type="button"
2600		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2601		- data-source-url="<?php echo esc_attr($source_url); ?>"
2602		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2603		- data-data-source="<?php echo esc_attr($data_source); ?>"
2604		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2605		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2606		- style="color: var(--mxch-error);"
2607		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2608		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2609		- </button>
2610		- </td>
2611		- </tr>
2612		- <?php
2613		- // Render hidden chunk rows
2614		- foreach ($group as $chunk_index => $chunk) {
2615		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
2616		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
2617		- $content = isset($chunk->display_content) ? $chunk->display_content : (isset($chunk->article_content) ? $chunk->article_content : '');
2618		- $content_preview = mb_strlen($content) > $preview_length
2619		- ? mb_substr($content, 0, $preview_length) . '...'
2620		- : $content;
2621		- ?>
2622		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
2623		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
2624		- data-source="<?php echo esc_attr($data_source); ?>"
2625		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
2626		- <td style="padding: 12px 16px; text-align: center;">
2627		- <!-- Checkbox column placeholder for chunks (managed by group) -->
2628		- </td>
2629		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
2630		- <!-- Hidden ID column for chunks -->
2631		- </td>
2632		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2633		- <div class="mxchat-accordion-wrapper">
2634		- <div class="mxchat-content-preview">
2635		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
2636		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
2637		- </span>
2638		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2639		- <?php if (mb_strlen($content) > $preview_length) : ?>
2640		- <button class="mxchat-expand-toggle" type="button">
2641		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2642		- </button>
2643		- <?php endif; ?>
2644		- </div>
2645		- <div class="mxchat-content-full" style="display: none;">
2646		- <div class="content-view">
2647		- <?php
2648		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2649		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2650		- echo wp_kses_post(wpautop($content));
2651		- echo '</div>';
2652		- } else {
2653		- echo wp_kses_post(wpautop($content));
2654		- }
2655		- ?>
2656		- </div>
2657		- </div>
2658		- </div>
2659		- </td>
2660		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2661		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
2662		- </td>
2663		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
2664		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
2665		- </td>
2666		- </tr>
2667		- <?php
2668		- }
2669		- } else {
2670		- // Single entry - display normally with accordion
2671		- $prompt = $first_prompt;
2672		- $content = isset($prompt->display_content) ? $prompt->display_content : (isset($prompt->article_content) ? $prompt->article_content : '');
2673		- $content_preview = mb_strlen($content) > $preview_length
2674		- ? mb_substr($content, 0, $preview_length) . '...'
2675		- : $content;
2676		- ?>
2677		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
2678		- data-source="<?php echo esc_attr($data_source); ?>"
2679		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2680		- <td style="padding: 12px 16px; text-align: center;">
2681		- <input type="checkbox"
2682		- class="mxchat-entry-checkbox"
2683		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2684		- data-source="<?php echo esc_attr($data_source); ?>"
2685		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
2686		- data-is-group="false"
2687		- data-chunk-count="1">
2688		- </td>
2689		- <td style="padding: 12px 16px; font-size: 13px;">
2690		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2691		- </td>
2692		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2693		- <div class="mxchat-accordion-wrapper">
2694		- <div class="mxchat-content-preview">
2695		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2696		- <?php if (mb_strlen($content) > $preview_length) : ?>
2697		- <button class="mxchat-expand-toggle" type="button">
2698		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2699		- </button>
2700		- <?php endif; ?>
2701		- </div>
2702		- <div class="mxchat-content-full" style="display: none;">
2703		- <div class="content-view">
2704		- <?php
2705		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2706		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2707		- echo wp_kses_post(wpautop($content));
2708		- echo '</div>';
2709		- } else {
2710		- echo wp_kses_post(wpautop($content));
2711		- }
2712		- ?>
2713		- </div>
2714		- </div>
2715		- </div>
2716		- </td>
2717		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2718		- <?php
2719		- $actual_source = $source_url;
2720		- if (strpos($source_url, '_ungrouped_') === 0) {
2721		- $actual_source = $prompt->source_url ?? '';
2722		- }
2723		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
2724		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2725		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2726		- <?php esc_html_e('View', 'mxchat'); ?>
2727		- </a>
2728		- <?php else : ?>
2729		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
2730		- <?php endif; ?>
2731		- </td>
2732		- <td style="padding: 12px 16px;">
2733		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-ajax" data-vector-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_pinecone_prompt_nonce'); ?>" style="color: var(--mxch-error);">
2734		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2735		- </button>
2736		- </td>
2737		- </tr>
2738		- <?php
2739		- }
2740		- }
2741		- }
2742		- $html = ob_get_clean();
2743		-
2744		- // Generate pagination HTML for Pinecone
2745		- $total_pages = ceil($total_records / $per_page);
2746		- $pagination_html = '';
2747		- if ($total_pages > 1) {
2748		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
2749		-
2750		- // Previous button
2751		- if ($page > 1) {
2752		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
2753		- }
2754		-
2755		- // Page numbers
2756		- $start_page = max(1, $page - 2);
2757		- $end_page = min($total_pages, $page + 2);
2758		-
2759		- if ($start_page > 1) {
2760		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
2761		- if ($start_page > 2) {
2762		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2763		- }
2764		- }
2765		-
2766		- for ($i = $start_page; $i <= $end_page; $i++) {
2767		- if ($i == $page) {
2768		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
2769		- } else {
2770		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
2771		- }
2772		- }
2773		-
2774		- if ($end_page < $total_pages) {
2775		- if ($end_page < $total_pages - 1) {
2776		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2777		- }
2778		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
2779		- }
2780		-
2781		- // Next button
2782		- if ($page < $total_pages) {
2783		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
2784		- }
2785		-
2786		- $pagination_html .= '</div>';
2787		- }
2788		-
2789		- wp_send_json_success(array(
2790		- 'html' => $html,
2791		- 'pagination_html' => $pagination_html,
2792		- 'total_count' => $total_records,
2793		- 'total_pages' => $total_pages,
2794		- 'page' => $page,
2795		- 'per_page' => $per_page,
2796		- 'data_source' => 'pinecone'
2797		- ));
2798		-}
2799		-
2800		-/**
2801		- * AJAX handler for pagination - handles both WordPress DB and Pinecone data sources
2802		- * Returns paginated entries without requiring a full page reload
2803		- */
2804		-public function ajax_mxchat_paginate_entries() {
2805		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2806		-
2807		- if (!current_user_can('manage_options')) {
2808		- wp_send_json_error(array('message' => 'Unauthorized'));
2809		- return;
2810		- }
2811		-
2812		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2813		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2814		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2815		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2816		- $per_page = 25;
2817		-
2818		- // Check if Pinecone is enabled for this bot
2819		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
2820		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
2821		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2822		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
2823		-
2824		- if ($use_pinecone && $has_pinecone_api) {
2825		- // Delegate to Pinecone pagination handler (pass search params)
2826		- $_POST['page'] = $page;
2827		- $_POST['search'] = $search_query;
2828		- $_POST['content_type'] = $content_type_filter;
2829		- $this->ajax_mxchat_refresh_pinecone_entries();
2830		- return;
2831		- }
2832		-
2833		- // WordPress DB pagination - MUST match initial page load logic exactly
2834		- global $wpdb;
2835		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2836		- $offset = ($page - 1) * $per_page;
2837		-
2838		- // Build WHERE clause for search and content type filtering
2839		- $where_clauses = array();
2840		- $where_values = array();
2841		-
2842		- if ($search_query) {
2843		- $where_clauses[] = "article_content LIKE %s";
2844		- $where_values[] = '%' . $wpdb->esc_like($search_query) . '%';
2845		- }
2846		-
2847		- if ($content_type_filter) {
2848		- switch ($content_type_filter) {
2849		- case 'manual':
2850		- $where_clauses[] = "(source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')";
2851		- break;
2852		- case 'pdf':
2853		- $where_clauses[] = "source_url LIKE '%.pdf'";
2854		- break;
2855		- case 'url':
2856		- $where_clauses[] = "source_url != '' AND source_url IS NOT NULL AND source_url NOT LIKE 'mxchat://%' AND source_url NOT LIKE '%.pdf'";
2857		- break;
2858		- }
2859		- }
2860		-
2861		- $where_sql = !empty($where_clauses) ? 'WHERE ' . implode(' AND ', $where_clauses) : '';
2862		-
2863		- // Count grouped entries with filters applied
2864		- if (!empty($where_values)) {
2865		- $count_args = array_merge($where_values, $where_values);
2866		- $count_query = $wpdb->prepare(
2867		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2868		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))",
2869		- ...$count_args
2870		- );
2871		- $total_records = $wpdb->get_var($count_query);
2872		- } else if (!empty($where_sql)) {
2873		- // Content type filter only (no search), no prepared values needed
2874		- $total_records = $wpdb->get_var(
2875		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2876		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))"
2877		- );
2878		- } else {
2879		- // No filters
2880		- $total_records = $wpdb->get_var(
2881		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2882		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
2883		- );
2884		- }
2885		- $total_pages = ceil($total_records / $per_page);
2886		-
2887		- // Step 1: Get unique source_urls for this page (ordered by latest timestamp) with filters
2888		- if (!empty($where_values)) {
2889		- $query_args = array_merge($where_values, array($per_page, $offset));
2890		- $urls_query = $wpdb->prepare(
2891		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2892		- {$where_sql}
2893		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2894		- ...$query_args
2895		- );
2896		- } else if (!empty($where_sql)) {
2897		- $urls_query = $wpdb->prepare(
2898		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2899		- {$where_sql}
2900		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2901		- $per_page, $offset
2902		- );
2903		- } else {
2904		- $urls_query = $wpdb->prepare(
2905		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2906		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2907		- $per_page, $offset
2908		- );
2909		- }
2910		- $page_urls = $wpdb->get_results($urls_query);
2911		-
2912		- // Step 2: Build list of source_urls to fetch
2913		- $url_list = array();
2914		- $url_order_map = array();
2915		- $order_index = 0;
2916		- foreach ($page_urls as $url_row) {
2917		- $url = $url_row->source_url;
2918		- $url_list[] = $url;
2919		- $url_order_map[$url] = $order_index++;
2920		- }
2921		-
2922		- // Step 3: Fetch all rows for these source_urls (with search filter if applicable)
2923		- $prompts = array();
2924		- if (!empty($url_list)) {
2925		- $placeholders = implode(',', array_fill(0, count($url_list), '%s'));
2926		- if ($search_query) {
2927		- // Include search filter in the final fetch
2928		- $prompts_query = $wpdb->prepare(
2929		- "SELECT id, article_content, source_url, timestamp, role_restriction
2930		- FROM {$table_name}
2931		- WHERE source_url IN ($placeholders) AND article_content LIKE %s
2932		- ORDER BY timestamp DESC",
2933		- ...array_merge($url_list, ['%' . $wpdb->esc_like($search_query) . '%'])
2934		- );
2935		- } else {
2936		- $prompts_query = $wpdb->prepare(
2937		- "SELECT id, article_content, source_url, timestamp, role_restriction
2938		- FROM {$table_name}
2939		- WHERE source_url IN ($placeholders)
2940		- ORDER BY timestamp DESC",
2941		- $url_list
2942		- );
2943		- }
2944		- $prompts = $wpdb->get_results($prompts_query);
2945		- }
2946		-
2947		- // Group prompts by source_url for chunk display
2948		- $grouped_prompts = array();
2949		- foreach ($prompts as $prompt) {
2950		- $source_url = $prompt->source_url ?? '';
2951		-
2952		- // Parse chunk metadata using the proper chunker method (same as initial page load)
2953		- if (class_exists('MxChat_Chunker')) {
2954		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($prompt->article_content);
2955		- $prompt->chunk_metadata = $chunk_meta['metadata'];
2956		- $prompt->display_content = $chunk_meta['text'];
2957		- } else {
2958		- $prompt->chunk_metadata = array();
2959		- $prompt->display_content = $prompt->article_content;
2960		- }
2961		-
2962		- if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0) {
2963		- if (!isset($grouped_prompts[$source_url])) {
2964		- $grouped_prompts[$source_url] = array();
2965		- }
2966		- $grouped_prompts[$source_url][] = $prompt;
2967		- } else {
2968		- // Ungrouped entries
2969		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2970		- }
2971		- }
2972		-
2973		- // Sort groups by the original URL order (newest first)
2974		- uksort($grouped_prompts, function($a, $b) use ($url_order_map) {
2975		- $order_a = isset($url_order_map[$a]) ? $url_order_map[$a] : PHP_INT_MAX;
2976		- $order_b = isset($url_order_map[$b]) ? $url_order_map[$b] : PHP_INT_MAX;
2977		- return $order_a - $order_b;
2978		- });
2979		-
2980		- // Sort each group internally by chunk_index
2981		- foreach ($grouped_prompts as $source_url => &$group) {
2982		- usort($group, function($a, $b) {
2983		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2984		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2985		- return $index_a - $index_b;
2986		- });
2987		- }
2988		- unset($group);
2989		-
2990		- // Build HTML for the table rows
2991		- ob_start();
2992		- $display_index = 0;
2993		- $current_page = $page;
2994		- $data_source = 'wordpress';
2995		- $current_bot_id = $bot_id;
2996		- $preview_length = 150;
2997		-
2998		- if (empty($grouped_prompts)) {
2999		- echo '<tr><td colspan="5" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
3000		- esc_html_e('No knowledge entries found. Use the Import Options to add content.', 'mxchat');
3001		- echo '</td></tr>';
3002		- } else {
3003		- foreach ($grouped_prompts as $source_url => $group) {
3004		- $chunk_count = count($group);
3005		- $first_prompt = $group[0];
3006		- $display_index++;
3007		-
3008		- if ($chunk_count > 1) {
3009		- // Multiple chunks - show grouped row with expand button
3010		- $group_id = 'group-' . md5($source_url);
3011		- ?>
3012		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
3013		- class="mxchat-chunk-group-header"
3014		- data-source="<?php echo esc_attr($data_source); ?>"
3015		- data-group-id="<?php echo esc_attr($group_id); ?>"
3016		- style="border-bottom: 1px solid var(--mxch-card-border);">
3017		- <td style="padding: 12px 16px; text-align: center;">
3018		- <input type="checkbox"
3019		- class="mxchat-entry-checkbox"
3020		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
3021		- data-source="<?php echo esc_attr($data_source); ?>"
3022		- data-source-url="<?php echo esc_attr($source_url); ?>"
3023		- data-is-group="true"
3024		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
3025		- </td>
3026		- <td style="padding: 12px 16px; font-size: 13px;">
3027		- <?php echo esc_html($first_prompt->id); ?>
3028		- </td>
3029		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3030		- <div class="mxchat-chunk-group-info">
3031		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
3032		- <span class="dashicons dashicons-arrow-right-alt2"></span>
3033		- </button>
3034		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
3035		- <span class="mxchat-chunk-preview">
3036		- <?php
3037		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : $first_prompt->article_content;
3038		- $content_preview = mb_substr($parent_content, 0, 100);
3039		- echo esc_html($content_preview . '...');
3040		- ?>
3041		- </span>
3042		- </div>
3043		- </td>
3044		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3045		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
3046		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
3047		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
3048		- <?php esc_html_e('View Source', 'mxchat'); ?>
3049		- </a>
3050		- <?php else : ?>
3051		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
3052		- <?php endif; ?>
3053		- </td>
3054		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
3055		- <?php if ($data_source !== 'pinecone') : ?>
3056		- <button type="button"
3057		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
3058		- data-source-url="<?php echo esc_attr($source_url); ?>"
3059		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
3060		- data-data-source="<?php echo esc_attr($data_source); ?>"
3061		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3062		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
3063		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
3064		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
3065		- </button>
3066		- <?php endif; ?>
3067		- <button type="button"
3068		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
3069		- data-source-url="<?php echo esc_attr($source_url); ?>"
3070		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
3071		- data-data-source="<?php echo esc_attr($data_source); ?>"
3072		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3073		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
3074		- style="color: var(--mxch-error);"
3075		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
3076		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
3077		- </button>
3078		- </td>
3079		- </tr>
3080		- <?php
3081		- // Render hidden chunk rows
3082		- foreach ($group as $chunk_index => $chunk) {
3083		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
3084		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
3085		- $content = isset($chunk->display_content) ? $chunk->display_content : $chunk->article_content;
3086		- $content_preview = mb_strlen($content) > $preview_length
3087		- ? mb_substr($content, 0, $preview_length) . '...'
3088		- : $content;
3089		- ?>
3090		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
3091		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
3092		- data-source="<?php echo esc_attr($data_source); ?>"
3093		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
3094		- <td style="padding: 12px 16px; text-align: center;">
3095		- <!-- Checkbox column placeholder for chunks (managed by group) -->
3096		- </td>
3097		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
3098		- <!-- Hidden ID column for chunks -->
3099		- </td>
3100		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3101		- <div class="mxchat-accordion-wrapper">
3102		- <div class="mxchat-content-preview">
3103		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
3104		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
3105		- </span>
3106		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
3107		- <?php if (mb_strlen($content) > $preview_length) : ?>
3108		- <button class="mxchat-expand-toggle" type="button">
3109		- <span class="dashicons dashicons-arrow-down-alt2"></span>
3110		- </button>
3111		- <?php endif; ?>
3112		- </div>
3113		- <div class="mxchat-content-full" style="display: none;">
3114		- <div class="content-view">
3115		- <?php
3116		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
3117		- echo '<div dir="rtl" lang="he" class="rtl-content">';
3118		- echo wp_kses_post(wpautop($content));
3119		- echo '</div>';
3120		- } else {
3121		- echo wp_kses_post(wpautop($content));
3122		- }
3123		- ?>
3124		- </div>
3125		- </div>
3126		- </div>
3127		- </td>
3128		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3129		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
3130		- </td>
3131		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
3132		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
3133		- </td>
3134		- </tr>
3135		- <?php
3136		- }
3137		- } else {
3138		- // Single entry - display normally with accordion
3139		- $prompt = $first_prompt;
3140		- $content = isset($prompt->display_content) ? $prompt->display_content : $prompt->article_content;
3141		- $content_preview = mb_strlen($content) > $preview_length
3142		- ? mb_substr($content, 0, $preview_length) . '...'
3143		- : $content;
3144		- ?>
3145		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
3146		- data-source="<?php echo esc_attr($data_source); ?>"
3147		- style="border-bottom: 1px solid var(--mxch-card-border);">
3148		- <td style="padding: 12px 16px; text-align: center;">
3149		- <input type="checkbox"
3150		- class="mxchat-entry-checkbox"
3151		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
3152		- data-source="<?php echo esc_attr($data_source); ?>"
3153		- data-source-url="<?php echo esc_attr($source_url); ?>"
3154		- data-is-group="false">
3155		- </td>
3156		- <td style="padding: 12px 16px; font-size: 13px;">
3157		- <?php echo esc_html($prompt->id); ?>
3158		- </td>
3159		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3160		- <div class="mxchat-accordion-wrapper">
3161		- <div class="mxchat-content-preview">
3162		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
3163		- <?php if (mb_strlen($content) > $preview_length) : ?>
3164		- <button class="mxchat-expand-toggle" type="button">
3165		- <span class="dashicons dashicons-arrow-down-alt2"></span>
3166		- </button>
3167		- <?php endif; ?>
3168		- </div>
3169		- <div class="mxchat-content-full" style="display: none;">
3170		- <div class="content-view">
3171		- <?php
3172		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
3173		- echo '<div dir="rtl" lang="he" class="rtl-content">';
3174		- echo wp_kses_post(wpautop($content));
3175		- echo '</div>';
3176		- } else {
3177		- echo wp_kses_post(wpautop($content));
3178		- }
3179		- ?>
3180		- </div>
3181		- </div>
3182		- </div>
3183		- </td>
3184		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3185		- <?php
3186		- $actual_source = $source_url;
3187		- if (strpos($source_url, '_ungrouped_') === 0) {
3188		- $actual_source = $prompt->source_url ?? '';
3189		- }
3190		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
3191		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
3192		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
3193		- <?php esc_html_e('View', 'mxchat'); ?>
3194		- </a>
3195		- <?php else : ?>
3196		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
3197		- <?php endif; ?>
3198		- </td>
3199		- <td style="padding: 12px 16px; white-space: nowrap;">
3200		- <button type="button"
3201		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
3202		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
3203		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
3204		- data-data-source="<?php echo esc_attr($data_source); ?>"
3205		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3206		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
3207		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
3208		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
3209		- </button>
3210		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-wordpress" data-entry-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_wordpress_prompt_nonce'); ?>" style="color: var(--mxch-error);">
3211		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
3212		- </button>
3213		- </td>
3214		- </tr>
3215		- <?php
3216		- }
3217		- }
3218		- }
3219		- $html = ob_get_clean();
3220		-
3221		- // Generate pagination HTML (include search/filter data for subsequent pages)
3222		- $pagination_html = '';
3223		- if ($total_pages > 1) {
3224		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
3225		-
3226		- // Previous button
3227		- if ($page > 1) {
3228		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
3229		- }
3230		-
3231		- // Page numbers
3232		- $start_page = max(1, $page - 2);
3233		- $end_page = min($total_pages, $page + 2);
3234		-
3235		- if ($start_page > 1) {
3236		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
3237		- if ($start_page > 2) {
3238		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
3239		- }
3240		- }
3241		-
3242		- for ($i = $start_page; $i <= $end_page; $i++) {
3243		- if ($i == $page) {
3244		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
3245		- } else {
3246		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
3247		- }
3248		- }
3249		-
3250		- if ($end_page < $total_pages) {
3251		- if ($end_page < $total_pages - 1) {
3252		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
3253		- }
3254		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
3255		- }
3256		-
3257		- // Next button
3258		- if ($page < $total_pages) {
3259		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
3260		- }
3261		-
3262		- $pagination_html .= '</div>';
3263		- }
3264		-
3265		- wp_send_json_success(array(
3266		- 'html' => $html,
3267		- 'pagination_html' => $pagination_html,
3268		- 'total_count' => $total_records,
3269		- 'total_pages' => $total_pages,
3270		- 'page' => $page,
3271		- 'per_page' => $per_page,
3272		- 'data_source' => 'wordpress'
3273		- ));
3274		-}
3275		-
3276		-/**
3277		- * AJAX handler to detect available sitemaps on the site
3278		- * Optimized for speed - only checks primary sitemap indexes first
3279		- */
3280		-public function ajax_mxchat_detect_sitemaps() {
3281		- check_ajax_referer('mxchat_detect_sitemaps_nonce', 'nonce');
3282		-
3283		- if (!current_user_can('manage_options')) {
3284		- wp_send_json_error(array('message' => 'Unauthorized'));
3285		- return;
3286		- }
3287		-
3288		- $site_url = get_site_url();
3289		- $sitemaps = array();
3290		- $found_index = false;
3291		-
3292		- // Only check the main sitemap index files first (much faster)
3293		- // These are the primary entry points that contain sub-sitemaps
3294		- $primary_indexes = array(
3295		- 'sitemap_index.xml' => 'Yoast SEO / Rank Math', // Yoast & Rank Math
3296		- 'wp-sitemap.xml' => 'WordPress Core', // WordPress Core
3297		- 'sitemap.xml' => 'Standard', // Generic/AIOSEO
3298		- );
3299		-
3300		- foreach ($primary_indexes as $path => $source) {
3301		- $url = trailingslashit($site_url) . $path;
3302		-
3303		- $response = wp_remote_head($url, array(
3304		- 'timeout' => 10,
3305		- 'sslverify' => false,
3306		- 'redirection' => 1,
3307		- 'user-agent' => mxchat_ingest_user_agent(),
3308		- ));
3309		-
3310		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3311		- // Found a sitemap index - parse it to get sub-sitemaps
3312		- $sub_sitemaps = $this->parse_sitemap_index($url);
3313		- if (!empty($sub_sitemaps)) {
3314		- $sitemaps[] = array(
3315		- 'url' => $url,
3316		- 'type' => 'index',
3317		- 'source' => $source,
3318		- 'sub_sitemaps' => $sub_sitemaps
3319		- );
3320		- $found_index = true;
3321		- // Found a valid index, no need to check others
3322		- break;
3323		- }
3324		- }
3325		- }
3326		-
3327		- // If no sitemap index found, check for standalone sitemaps
3328		- if (!$found_index) {
3329		- $standalone_sitemaps = array(
3330		- 'post-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
3331		- 'page-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
3332		- );
3333		-
3334		- foreach ($standalone_sitemaps as $path => $info) {
3335		- $url = trailingslashit($site_url) . $path;
3336		-
3337		- $response = wp_remote_head($url, array(
3338		- 'timeout' => 2,
3339		- 'sslverify' => false
3340		- ));
3341		-
3342		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3343		- $sitemaps[] = array(
3344		- 'url' => $url,
3345		- 'type' => $info['type'],
3346		- 'source' => $info['source'],
3347		- 'url_count' => 0 // Skip URL count for speed
3348		- );
3349		- }
3350		- }
3351		- }
3352		-
3353		- wp_send_json_success(array(
3354		- 'sitemaps' => $sitemaps,
3355		- 'site_url' => $site_url
3356		- ));
3357		-}
3358		-
3359		-/**
3360		- * Parse a sitemap index to get sub-sitemaps
3361		- * Optimized: doesn't fetch URL count for each sub-sitemap (too slow)
3362		- */
3363		-private function parse_sitemap_index($url) {
3364		- $sub_sitemaps = array();
3365		-
3366		- $response = wp_remote_get($url, array(
3367		- 'timeout' => 30,
3368		- 'sslverify' => false,
3369		- 'user-agent' => mxchat_ingest_user_agent(),
3370		- 'headers' => array(
3371		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
3372		- ),
3373		- ));
3374		-
3375		- if (is_wp_error($response)) {
3376		- return $sub_sitemaps;
3377		- }
3378		-
3379		- $body = wp_remote_retrieve_body($response);
3380		- if (empty($body)) {
3381		- return $sub_sitemaps;
3382		- }
3383		-
3384		- // Suppress XML errors
3385		- libxml_use_internal_errors(true);
3386		- $xml = simplexml_load_string($body);
3387		- libxml_clear_errors();
3388		-
3389		- if ($xml === false) {
3390		- return $sub_sitemaps;
3391		- }
3392		-
3393		- // Check if it's a sitemap index (contains <sitemap> elements)
3394		- if (isset($xml->sitemap)) {
3395		- foreach ($xml->sitemap as $sitemap) {
3396		- $loc = (string) $sitemap->loc;
3397		- if (!empty($loc)) {
3398		- // Try to determine the type from the URL
3399		- $type = 'content';
3400		- if (strpos($loc, 'category') !== false || strpos($loc, 'tag') !== false || strpos($loc, 'taxonomy') !== false) {
3401		- $type = 'taxonomy';
3402		- } elseif (strpos($loc, 'author') !== false || strpos($loc, 'user') !== false) {
3403		- $type = 'author';
3404		- }
3405		-
3406		- // Skip URL count - too slow to fetch for each sitemap
3407		- $sub_sitemaps[] = array(
3408		- 'url' => $loc,
3409		- 'type' => $type,
3410		- 'url_count' => 0, // Don't fetch - takes too long
3411		- 'name' => basename(parse_url($loc, PHP_URL_PATH))
3412		- );
3413		- }
3414		- }
3415		- }
3416		-
3417		- return $sub_sitemaps;
3418		-}
3419		-
3420		-/**
3421		- * Get URL count from a sitemap
3422		- */
3423		-private function get_sitemap_url_count($url) {
3424		- $response = wp_remote_get($url, array(
3425		- 'timeout' => 30,
3426		- 'sslverify' => false,
3427		- 'user-agent' => mxchat_ingest_user_agent(),
3428		- 'headers' => array(
3429		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
3430		- ),
3431		- ));
3432		-
3433		- if (is_wp_error($response)) {
3434		- return 0;
3435		- }
3436		-
3437		- $body = wp_remote_retrieve_body($response);
3438		- if (empty($body)) {
3439		- return 0;
3440		- }
3441		-
3442		- // Count <url> or <loc> elements
3443		- $count = preg_match_all('/<url>/i', $body, $matches);
3444		- return $count ?: 0;
3445		-}
3446		-
3447		-/**
3448		- * Get sitemaps declared in robots.txt
3449		- */
3450		-private function get_sitemaps_from_robots($site_url) {
3451		- $sitemaps = array();
3452		- $robots_url = trailingslashit($site_url) . 'robots.txt';
3453		-
3454		- $response = wp_remote_get($robots_url, array(
3455		- 'timeout' => 15,
3456		- 'sslverify' => false,
3457		- 'user-agent' => mxchat_ingest_user_agent(),
3458		- ));
3459		-
3460		- if (is_wp_error($response)) {
3461		- return $sitemaps;
3462		- }
3463		-
3464		- $body = wp_remote_retrieve_body($response);
3465		- if (empty($body)) {
3466		- return $sitemaps;
3467		- }
3468		-
3469		- // Find Sitemap: declarations
3470		- if (preg_match_all('/^Sitemap:\s*(.+)$/mi', $body, $matches)) {
3471		- foreach ($matches[1] as $sitemap_url) {
3472		- $sitemap_url = trim($sitemap_url);
3473		- if (filter_var($sitemap_url, FILTER_VALIDATE_URL)) {
3474		- $sitemaps[] = $sitemap_url;
3475		- }
3476		- }
3477		- }
3478		-
3479		- return $sitemaps;
3480		-}
3481		-
3482	1777	public function mxchat_stop_processing() {
3483	1778	// Verify permissions
3484	1779	if (!current_user_can('manage_options')) {
3485	1780	wp_die(esc_html__('Unauthorized access', 'mxchat'));
		@@ -3487,52 +1782,28 @@
3487	1782
3488	1783	// Verify nonce
3489	1784	check_admin_referer('mxchat_stop_processing_action', 'mxchat_stop_processing_nonce');
3490	1785
3491		- global $wpdb;
3492		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
3493		-
3494		- // Get active queue IDs
3495		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
3496		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
3497		-
3498		- // Delete all pending items from active queues
3499		- if ($sitemap_queue_id) {
3500		- $wpdb->delete(
3501		- $table_name,
3502		- array(
3503		- 'queue_id' => $sitemap_queue_id,
3504		- 'status' => 'pending'
3505		- ),
3506		- array('%s', '%s')
3507		- );
3508		-
3509		- delete_transient('mxchat_active_queue_sitemap');
	1786	+ // Get the last sitemap URL and clear its transient
	1787	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
	1788	+ if ($sitemap_url) {
	1789	+ delete_transient('mxchat_sitemap_status_' . md5($sitemap_url));
3510	1790	delete_transient('mxchat_last_sitemap_url');
3511	1791	}
3512		-
3513		- if ($pdf_queue_id) {
3514		- // Get PDF path before deleting
3515		- $pdf_path = $this->mxchat_get_queue_meta($pdf_queue_id, 'pdf_path');
3516		-
3517		- $wpdb->delete(
3518		- $table_name,
3519		- array(
3520		- 'queue_id' => $pdf_queue_id,
3521		- 'status' => 'pending'
3522		- ),
3523		- array('%s', '%s')
3524		- );
3525		-
3526		- // Delete PDF file
3527		- if ($pdf_path && file_exists($pdf_path)) {
3528		- wp_delete_file($pdf_path);
3529		- }
3530		-
3531		- delete_transient('mxchat_active_queue_pdf');
	1792	+
	1793	+ // Get the last PDF URL and clear its transient
	1794	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1795	+ if ($pdf_url) {
	1796	+ delete_transient('mxchat_pdf_status_' . md5($pdf_url));
3532	1797	delete_transient('mxchat_last_pdf_url');
3533	1798	}
3534	1799
	1800	+ // Unschedule any pending sitemap events
	1801	+ $timestamp = wp_next_scheduled('mxchat_process_sitemap_urls');
	1802	+ if ($timestamp) {
	1803	+ wp_unschedule_event($timestamp, 'mxchat_process_sitemap_urls');
	1804	+ }
	1805	+
3535	1806	// Redirect back with a success message
3536	1807	set_transient('mxchat_admin_notice_success',
3537	1808	esc_html__('Processing has been stopped successfully.', 'mxchat'),
3538	1809	30
		@@ -3539,12 +1810,8 @@
3539	1810	);
3540	1811	wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
3541	1812	exit;
3542	1813	}
3543		-
3544		-/**
3545		- * Get content list for processing
3546		- */
3547	1814	public function ajax_mxchat_get_content_list() {
3548	1815	// Verify the nonce
3549	1816	check_ajax_referer('mxchat_content_selector_nonce', 'nonce');
3550	1817
		@@ -3552,9 +1819,9 @@
3552	1819	wp_send_json_error(__('Unauthorized access', 'mxchat'));
3553	1820	}
3554	1821
3555	1822	$page = isset($_GET['page']) ? absint($_GET['page']) : 1;
3556		- $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 100;
	1823	+ $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 20;
3557	1824	$search = isset($_GET['search']) ? sanitize_text_field($_GET['search']) : '';
3558	1825	$post_type = isset($_GET['post_type']) ? sanitize_text_field($_GET['post_type']) : 'all';
3559	1826	$post_status = isset($_GET['post_status']) ? sanitize_text_field($_GET['post_status']) : 'publish';
3560	1827	$processed_filter = isset($_GET['processed_filter']) ? sanitize_text_field($_GET['processed_filter']) : 'all';
		@@ -3567,47 +1834,20 @@
3567	1834	'orderby' => 'date',
3568	1835	'order' => 'DESC',
3569	1836	);
3570	1837
3571		- // Handle post types - IMPROVED VERSION
	1838	+ // Handle post types
3572	1839	if ($post_type !== 'all') {
3573	1840	$args['post_type'] = $post_type;
3574	1841	} else {
3575		- // Get all available post types that might contain content
3576		- $all_post_types = array();
	1842	+ // Default to post and page if we can't get post types
	1843	+ $args['post_type'] = array('post', 'page');
3577	1844
3578		- // First get all public post types
3579		- $public_types = get_post_types(array('public' => true), 'names');
3580		- $all_post_types = array_merge($all_post_types, $public_types);
3581		-
3582		- // Add common forum/community post types
3583		- $forum_types = array('topic', 'reply', 'forum', 'wpforo_topic', 'wpforo_post');
3584		- foreach ($forum_types as $forum_type) {
3585		- if (post_type_exists($forum_type)) {
3586		- $all_post_types[] = $forum_type;
3587		- }
	1845	+ // Try to get public post types
	1846	+ $public_types = $this->mxchat_get_public_post_types();
	1847	+ if (is_array($public_types) && !empty($public_types)) {
	1848	+ $args['post_type'] = array_keys($public_types);
3588	1849	}
3589		-
3590		- // Add other commonly used post types
3591		- $common_types = array('product', 'job_listing', 'event', 'portfolio');
3592		- foreach ($common_types as $common_type) {
3593		- if (post_type_exists($common_type)) {
3594		- $all_post_types[] = $common_type;
3595		- }
3596		- }
3597		-
3598		- // Remove duplicates and ensure we have at least some post types
3599		- $all_post_types = array_unique($all_post_types);
3600		-
3601		- if (empty($all_post_types)) {
3602		- // Fallback to basic post types
3603		- $all_post_types = array('post', 'page');
3604		- }
3605		-
3606		- $args['post_type'] = $all_post_types;
3607		-
3608		- // Debug logging to see what post types are being queried
3609		- //error_log('MxChat Debug: Querying post types: ' . implode(', ', $all_post_types));
3610	1850	}
3611	1851
3612	1852	if (!empty($search)) {
3613	1853	$args['s'] = $search;
		@@ -3612,58 +1852,37 @@
3612	1852	if (!empty($search)) {
3613	1853	$args['s'] = $search;
3614	1854	}
3615	1855
3616		- // Get processed data from storage
	1856	+ // ================================
	1857	+ // FIXED: Check only the ACTIVE storage method
	1858	+ // ================================
	1859	+
3617	1860	$processed_data = array();
3618	1861
	1862	+ $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
3619	1863	$pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	1864	+ $pinecone_manager->mxchat_refresh_after_new_content($pinecone_options);
3620	1865	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3621	1866
3622	1867	if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3623		- // Get fresh data from Pinecone - no caching
	1868	+ // ONLY check Pinecone if it's enabled
3624	1869	$processed_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
3625	1870	} else {
3626		- // WordPress DB checking with better URL matching for all post types
	1871	+ // ONLY check WordPress DB if Pinecone is not enabled
3627	1872	global $wpdb;
3628	1873	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3629		- $processed_items = $wpdb->get_results("SELECT id, source_url, article_content, timestamp FROM {$table_name}");
3630		-
3631		- // Group items by source_url to count chunks
3632		- $url_chunk_counts = array();
3633		- $url_latest_timestamp = array();
3634		- $url_first_id = array();
3635		-
	1874	+ $processed_items = $wpdb->get_results("SELECT id, source_url, timestamp FROM {$table_name}");
	1875	+
3636	1876	if (!empty($processed_items)) {
3637	1877	foreach ($processed_items as $item) {
3638		- $url = $item->source_url;
3639		- if (empty($url)) continue;
3640		-
3641		- // Count chunks per URL
3642		- if (!isset($url_chunk_counts[$url])) {
3643		- $url_chunk_counts[$url] = 0;
3644		- $url_latest_timestamp[$url] = $item->timestamp;
3645		- $url_first_id[$url] = $item->id;
3646		- }
3647		- $url_chunk_counts[$url]++;
3648		-
3649		- // Track latest timestamp
3650		- if (strtotime($item->timestamp) > strtotime($url_latest_timestamp[$url])) {
3651		- $url_latest_timestamp[$url] = $item->timestamp;
3652		- }
3653		- }
3654		-
3655		- // Now build processed_data with chunk counts
3656		- foreach ($url_chunk_counts as $url => $chunk_count) {
3657		- $post_id = $this->mxchat_url_to_post_id_improved($url);
3658		-
	1878	+ $post_id = url_to_postid($item->source_url);
3659	1879	if ($post_id) {
3660	1880	$processed_data[$post_id] = array(
3661		- 'db_id' => $url_first_id[$url],
3662		- 'timestamp' => $url_latest_timestamp[$url],
3663		- 'url' => $url,
3664		- 'source' => 'wordpress',
3665		- 'chunk_count' => $chunk_count
	1881	+ 'db_id' => $item->id,
	1882	+ 'timestamp' => $item->timestamp,
	1883	+ 'url' => $item->source_url,
	1884	+ 'source' => 'wordpress'
3666	1885	);
3667	1886	}
3668	1887	}
3669	1888	}
		@@ -3668,8 +1887,10 @@
3668	1887	}
3669	1888	}
3670	1889	}
3671	1890
	1891	+ // ================================
	1892	+
3672	1893	// Get processed IDs as a simple array for in_array checks
3673	1894	$processed_ids = array_keys($processed_data);
3674	1895
3675	1896	// Handle processed/unprocessed filter
		@@ -3711,14 +1932,8 @@
3711	1932	$db_record_id = $item_data['db_id'];
3712	1933	}
3713	1934	}
3714	1935
3715		- // Get chunk count for this item
3716		- $chunk_count = 0;
3717		- if ($is_processed && isset($processed_data[$id]['chunk_count'])) {
3718		- $chunk_count = intval($processed_data[$id]['chunk_count']);
3719		- }
3720		-
3721	1936	$content_items[] = array(
3722	1937	'id' => $id,
3723	1938	'title' => get_the_title(),
3724	1939	'permalink' => get_permalink(),
		@@ -3729,10 +1944,9 @@
3729	1944	'word_count' => $word_count,
3730	1945	'already_processed' => $is_processed,
3731	1946	'processed_date' => $processed_date,
3732	1947	'db_record_id' => $db_record_id,
3733		- 'data_source' => $data_source,
3734		- 'chunk_count' => $chunk_count
	1948	+ 'data_source' => $data_source
3735	1949	);
3736	1950	}
3737	1951	wp_reset_postdata();
3738	1952	}
		@@ -3748,178 +1962,8 @@
3748	1962	wp_send_json_success($response);
3749	1963	exit;
3750	1964	}
3751	1965
3752		-
3753		-/**
3754		- * This function handles various WooCommerce URL formats and permalink structures
3755		- */
3756		-private function mxchat_url_to_post_id_improved($url) {
3757		- // First try the standard WordPress function
3758		- $post_id = url_to_postid($url);
3759		-
3760		- if ($post_id > 0) {
3761		- return $post_id;
3762		- }
3763		-
3764		- // If that fails, try more aggressive URL matching
3765		- // Remove trailing slashes and query parameters for better matching
3766		- $clean_url = rtrim($url, '/');
3767		- $clean_url = strtok($clean_url, '?'); // Remove query parameters
3768		-
3769		- // Try again with cleaned URL
3770		- $post_id = url_to_postid($clean_url);
3771		- if ($post_id > 0) {
3772		- return $post_id;
3773		- }
3774		-
3775		- // For bbPress forum topics, try extracting slug from URL
3776		- if (strpos($url, '/topic/') !== false || strpos($url, '/forums/') !== false) {
3777		- // Handle bbPress URLs: /forums/topic/topic-name/
3778		- if (preg_match('/\/forums\/topic\/([^\/\?]+)/', $url, $matches)) {
3779		- $topic_slug = $matches[1];
3780		-
3781		- // Look up topic by slug
3782		- $topic = get_page_by_path($topic_slug, OBJECT, 'topic');
3783		- if ($topic) {
3784		- return $topic->ID;
3785		- }
3786		-
3787		- // Alternative method: query by post_name
3788		- global $wpdb;
3789		- $post_id = $wpdb->get_var($wpdb->prepare(
3790		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3791		- $topic_slug
3792		- ));
3793		-
3794		- if ($post_id) {
3795		- return intval($post_id);
3796		- }
3797		- }
3798		-
3799		- // Handle simpler topic URLs: /topic/topic-name/
3800		- if (preg_match('/\/topic\/([^\/\?]+)/', $url, $matches)) {
3801		- $topic_slug = $matches[1];
3802		-
3803		- global $wpdb;
3804		- $post_id = $wpdb->get_var($wpdb->prepare(
3805		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3806		- $topic_slug
3807		- ));
3808		-
3809		- if ($post_id) {
3810		- return intval($post_id);
3811		- }
3812		- }
3813		- }
3814		-
3815		- // For WooCommerce products
3816		- if (strpos($url, '/product/') !== false || strpos($url, 'product=') !== false) {
3817		- // Extract product slug from various URL formats
3818		- $product_slug = '';
3819		-
3820		- // Handle pretty permalinks: /product/product-name/
3821		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
3822		- $product_slug = $matches[1];
3823		- }
3824		- // Handle query parameters: ?product=product-name
3825		- elseif (preg_match('/[\?&]product=([^&]+)/', $url, $matches)) {
3826		- $product_slug = $matches[1];
3827		- }
3828		-
3829		- if (!empty($product_slug)) {
3830		- // Look up product by slug
3831		- $product = get_page_by_path($product_slug, OBJECT, 'product');
3832		- if ($product) {
3833		- return $product->ID;
3834		- }
3835		-
3836		- // Alternative method: query by post_name
3837		- global $wpdb;
3838		- $post_id = $wpdb->get_var($wpdb->prepare(
3839		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'product' AND post_status = 'publish'",
3840		- $product_slug
3841		- ));
3842		-
3843		- if ($post_id) {
3844		- return intval($post_id);
3845		- }
3846		- }
3847		- }
3848		-
3849		- // Generic approach: try to extract slug and match against all post types
3850		- $parsed_url = wp_parse_url($clean_url);
3851		- $path = $parsed_url['path'] ?? '';
3852		-
3853		- if (!empty($path)) {
3854		- // Get the last part of the path as potential slug
3855		- $path_parts = array_filter(explode('/', trim($path, '/')));
3856		- $potential_slug = end($path_parts);
3857		-
3858		- if (!empty($potential_slug)) {
3859		- global $wpdb;
3860		-
3861		- // Try to find any post with this slug
3862		- $post_id = $wpdb->get_var($wpdb->prepare(
3863		- "SELECT ID FROM {$wpdb->posts}
3864		- WHERE post_name = %s
3865		- AND post_status IN ('publish', 'closed', 'private')
3866		- AND post_type NOT IN ('revision', 'attachment', 'nav_menu_item')
3867		- ORDER BY CASE
3868		- WHEN post_type = 'post' THEN 1
3869		- WHEN post_type = 'page' THEN 2
3870		- WHEN post_type = 'topic' THEN 3
3871		- WHEN post_type = 'product' THEN 4
3872		- ELSE 5
3873		- END
3874		- LIMIT 1",
3875		- $potential_slug
3876		- ));
3877		-
3878		- if ($post_id) {
3879		- return intval($post_id);
3880		- }
3881		- }
3882		- }
3883		-
3884		- // ADDITIONAL: Try direct database lookup by URL variations
3885		- global $wpdb;
3886		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3887		-
3888		- // Try variations of the URL (with/without trailing slash, http/https)
3889		- $url_variations = array(
3890		- $url,
3891		- rtrim($url, '/'),
3892		- $url . '/',
3893		- str_replace('http://', 'https://', $url),
3894		- str_replace('https://', 'http://', $url),
3895		- str_replace('http://', 'https://', rtrim($url, '/')),
3896		- str_replace('https://', 'http://', rtrim($url, '/'))
3897		- );
3898		-
3899		- // Remove duplicates
3900		- $url_variations = array_unique($url_variations);
3901		-
3902		- foreach ($url_variations as $variation) {
3903		- $existing_record = $wpdb->get_row($wpdb->prepare(
3904		- "SELECT id, source_url FROM $table_name WHERE source_url = %s",
3905		- $variation
3906		- ));
3907		-
3908		- if ($existing_record) {
3909		- // Try to get post ID from this stored URL
3910		- $stored_post_id = url_to_postid($existing_record->source_url);
3911		- if ($stored_post_id > 0) {
3912		- return $stored_post_id;
3913		- }
3914		- }
3915		- }
3916		-
3917		- return 0; // No match found
3918		-}
3919		-/**
3920		- * Process selected content via AJAX
3921		- */
3922	1966	public function ajax_mxchat_process_selected_content() {
3923	1967	// Basic request validation
3924	1968	if (!check_ajax_referer('mxchat_content_selector_nonce', 'nonce', false)) {
3925	1969	wp_send_json_error('Invalid nonce');
		@@ -3943,16 +1987,8 @@
3943	1987	wp_send_json_error('No content selected');
3944	1988	exit;
3945	1989	}
3946	1990
3947		- // Get bot_id from request
3948		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
3949		-
3950		- // ACF→PDF extraction is an install-level setting (Knowledge → ACF Fields,
3951		- // plan 11720c). The import modal shows a passive status line pointing
3952		- // there; the old per-batch checkbox and its remembered default are gone.
3953		- $extract_acf_pdfs = get_option('mxchat_acf_pdf_extraction', '0') === '1';
3954		-
3955	1991	// Process only ONE post at a time to avoid request size issues
3956	1992	$post_id = reset($post_ids);
3957	1993	$post = get_post($post_id);
3958	1994
		@@ -3959,224 +1995,54 @@
3959	1995	if (!$post) {
3960	1996	wp_send_json_error('Post not found');
3961	1997	exit;
3962	1998	}
3963		-
3964		- /**
3965		- * Allow developers to modify post data before processing into the knowledge base.
3966		- * Applied on BOTH content-preparation paths (this manual bulk import and the
3967		- * auto-sync path in mxchat_handle_post_update) with the same signature, so a
3968		- * callback registered once covers every indexing route. Purely additive —
3969		- * zero behaviour change when unhooked.
3970		- *
3971		- * @param WP_Post $post The post about to be indexed.
3972		- * @param string $bot_id Bot context for this import.
3973		- */
3974		- $post = apply_filters('mxchat_before_process_post', $post, $bot_id);
3975		- if (!($post instanceof WP_Post)) {
3976		- $post = get_post($post_id); // defend against a bad callback return
	1999	+
	2000	+ // Get minimal content
	2001	+ $content = $post->post_title . "\n\n" . wp_strip_all_tags($post->post_content);
	2002	+ $content = substr($content, 0, 10000); // Limit content size
	2003	+
	2004	+ // Get API key with proper model detection
	2005	+ $options = get_option('mxchat_options');
	2006	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	2007	+
	2008	+ if (strpos($selected_model, 'voyage') === 0) {
	2009	+ $api_key = $options['voyage_api_key'] ?? '';
	2010	+ $provider_name = 'Voyage AI';
	2011	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	2012	+ $api_key = $options['gemini_api_key'] ?? '';
	2013	+ $provider_name = 'Google Gemini';
	2014	+ } else {
	2015	+ $api_key = $options['api_key'] ?? '';
	2016	+ $provider_name = 'OpenAI';
3977	2017	}
3978		-
3979		- // Get content including title, short description (for WooCommerce), and main content
3980		- $content = $post->post_title . "\n\n";
3981		-
3982		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
3983		- if (!empty($post->post_excerpt)) {
3984		- // Remove shortcode tags but preserve content inside them
3985		- $clean_excerpt = $this->strip_shortcode_tags_preserve_content($post->post_excerpt);
3986		- $content .= "Short Description: " . wp_strip_all_tags($clean_excerpt) . "\n\n";
3987		- }
3988		-
3989		- // Add main content - remove shortcode tags but preserve content inside them
3990		- $clean_content = $this->strip_shortcode_tags_preserve_content($post->post_content);
3991		- $content .= wp_strip_all_tags($clean_content);
3992		-
3993		- // ADD WOOCOMMERCE PRODUCT DATA (pricing, stock, categories, custom tabs)
3994		- if (get_post_type($post_id) === 'product' && class_exists('WooCommerce')) {
3995		- $product = wc_get_product($post_id);
3996		-
3997		- if ($product) {
3998		- // Get pricing information
3999		- $regular_price = $product->get_regular_price();
4000		- $sale_price = $product->get_sale_price();
4001		- $price = $product->get_price();
4002		- $sku = $product->get_sku();
4003		-
4004		- // Get currency symbol
4005		- $currency_symbol = get_woocommerce_currency_symbol();
4006		-
4007		- // Add pricing information
4008		- $content .= "\n";
4009		- if (!empty($regular_price)) {
4010		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
4011		- } elseif (!empty($price)) {
4012		- $content .= "Price: " . $currency_symbol . $price . "\n";
4013		- }
4014		-
4015		- if (!empty($sale_price) && $sale_price !== $regular_price) {
4016		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
4017		- }
4018		-
4019		- // Handle variable products - show price range
4020		- if ($product->is_type('variable')) {
4021		- $min_price = $product->get_variation_price('min');
4022		- $max_price = $product->get_variation_price('max');
4023		- if ($min_price !== $max_price) {
4024		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
4025		- }
4026		- }
4027		-
4028		- if (!empty($sku)) {
4029		- $content .= "SKU: " . $sku . "\n";
4030		- }
4031		-
4032		- // Get product categories
4033		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
4034		- if (!empty($categories) && !is_wp_error($categories)) {
4035		- $content .= "Categories: " . implode(', ', $categories) . "\n";
4036		- }
4037		- }
4038		-
4039		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
4040		- $custom_tabs = get_post_meta($post_id, 'yikes_woo_products_tabs', true);
4041		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
4042		- foreach ($custom_tabs as $tab) {
4043		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
4044		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
4045		-
4046		- if (!empty($tab_title) && !empty($tab_content)) {
4047		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
4048		- }
4049		- }
4050		- }
4051		-
4052		- // Also check for reusable/saved tabs applied to this product
4053		- $applied_saved_tabs = get_post_meta($post_id, 'yikes_woo_reusable_products_tabs_applied', true);
4054		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
4055		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
4056		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
4057		- foreach ($applied_saved_tabs as $saved_tab_id) {
4058		- if (isset($saved_tabs[$saved_tab_id])) {
4059		- $tab = $saved_tabs[$saved_tab_id];
4060		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
4061		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
4062		-
4063		- if (!empty($tab_title) && !empty($tab_content)) {
4064		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
4065		- }
4066		- }
4067		- }
4068		- }
4069		- }
4070		- }
4071		-
4072		- // ADD ACF FIELDS SUPPORT
4073		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
4074		- $pdf_extracted_count = 0;
4075		- if (!empty($acf_fields)) {
4076		- $acf_content_parts = array();
4077		- $pdf_attachment_ids = array();
4078		-
4079		- foreach ($acf_fields as $field_name => $field_value) {
4080		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
4081		-
4082		- if (!empty($formatted_value)) {
4083		- $field_label = ucwords(str_replace('_', ' ', $field_name));
4084		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
4085		- }
4086		-
4087		- // Walk this field's value tree for any PDF attachment references and queue them for extraction.
4088		- // Only when the user opted into ACF→PDF extraction for this batch; otherwise the ACF text
4089		- // still lands in the KB but the heavier PDF parsing is skipped.
4090		- if ($extract_acf_pdfs) {
4091		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
4092		- }
4093		- }
4094		-
4095		- if (!empty($acf_content_parts)) {
4096		- $content .= "\n\n" . implode("\n", $acf_content_parts);
4097		- }
4098		-
4099		- // Extract text from each unique PDF found in ACF fields and append as a labeled section
4100		- if ($extract_acf_pdfs && !empty($pdf_attachment_ids)) {
4101		- $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
4102		- $pdf_sections = array();
4103		- foreach ($pdf_attachment_ids as $att_id) {
4104		- $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
4105		- if (!empty($pdf_text)) {
4106		- $pdf_title = get_the_title($att_id);
4107		- $pdf_url = wp_get_attachment_url($att_id);
4108		- $header = 'PDF Attachment';
4109		- if (!empty($pdf_title)) {
4110		- $header .= ': ' . $pdf_title;
4111		- }
4112		- if (!empty($pdf_url)) {
4113		- $header .= ' (' . $pdf_url . ')';
4114		- }
4115		- $pdf_sections[] = $header . "\n" . $pdf_text;
4116		- $pdf_extracted_count++;
4117		- }
4118		- }
4119		- if (!empty($pdf_sections)) {
4120		- $content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
4121		- }
4122		- }
4123		- }
4124		-
4125		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
4126		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
4127		- if (!empty($custom_meta)) {
4128		- $meta_content_parts = array();
4129		-
4130		- foreach ($custom_meta as $meta_key => $meta_value) {
4131		- // Convert meta key to readable label
4132		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
4133		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
4134		- }
4135		-
4136		- if (!empty($meta_content_parts)) {
4137		- $content .= "\n\n" . implode("\n", $meta_content_parts);
4138		- }
4139		- }
4140		-
4141		- // Debug logging for WordPress Import content
4142		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Post ID: ' . $post_id . ' Title: ' . $post->post_title);
4143		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Raw post_content length: ' . strlen($post->post_content));
4144		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Final content length: ' . strlen($content));
4145		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Content preview: ' . substr($content, 0, 300));
4146		-
4147		- // Note: Removed 10,000 char limit - chunking now handles large content properly
4148		-
4149		- // Get bot-specific embedding decision — custom-provider-aware (plan cbd5fd)
4150		- $bot_options = $this->get_bot_options($bot_id);
4151		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
4152		-
4153		- $preflight = MxChat_Utils::embedding_preflight($options);
4154		- if (!$preflight['ok']) {
4155		- MxChat_Admin::mxchat_log_debug('api_error', $preflight['reason'] . ' (knowledge processing)');
4156		- wp_send_json_error($preflight['reason']);
	2018	+
	2019	+ if (empty($api_key)) {
	2020	+ wp_send_json_error($provider_name . ' API key not configured');
4157	2021	exit;
4158	2022	}
4159		- $api_key = $preflight['api_key'];
4160	2023
4161	2024	$source_url = get_permalink($post_id);
4162	2025	$vector_id = md5($source_url); // Vector ID for Pinecone
4163	2026
4164		- // Check for existing content in bot-specific storage
	2027	+ // ================================
	2028	+ // FIXED: Check for existing content in ONLY the active storage method
	2029	+ // ================================
	2030	+
4165	2031	$is_update = false;
4166	2032
4167		- // Get bot-specific Pinecone configuration
4168		- $bot_pinecone_config = $this->get_bot_pinecone_config($bot_id);
4169		- $use_pinecone = !empty($bot_pinecone_config) && ($bot_pinecone_config['use_pinecone'] ?? false);
	2033	+ // Check if Pinecone is enabled
	2034	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	2035	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
4170	2036
4171		- if ($use_pinecone && !empty($bot_pinecone_config['api_key'])) {
4172		- // Check Pinecone for this bot
4173		- $pinecone_data = $this->mxchat_get_pinecone_processed_content($bot_pinecone_config);
	2037	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2038	+ // ONLY check Pinecone if it's enabled
	2039	+ $pinecone_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
4174	2040	if (isset($pinecone_data[$post_id])) {
4175	2041	$is_update = true;
4176	2042	}
4177	2043	} else {
4178		- // Check WordPress DB (same as before since it's shared)
	2044	+ // ONLY check WordPress DB if Pinecone is not enabled
4179	2045	global $wpdb;
4180	2046	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
4181	2047	$existing_record = $wpdb->get_row($wpdb->prepare(
4182	2048	"SELECT id FROM $table_name WHERE source_url = %s",
		@@ -4186,54 +2052,54 @@
4186	2052	if ($existing_record) {
4187	2053	$is_update = true;
4188	2054	}
4189	2055	}
4190		-
4191		- // UPDATED 2.5.6: Determine content type based on post_type
4192		- $post_type = $post->post_type;
4193		- $content_type = 'content'; // Default fallback
4194		-
4195		- // Map WordPress post types to content types
4196		- switch ($post_type) {
4197		- case 'post':
4198		- $content_type = 'post';
4199		- break;
4200		- case 'page':
4201		- $content_type = 'page';
4202		- break;
4203		- case 'product':
4204		- $content_type = 'product';
4205		- break;
4206		- default:
4207		- // For custom post types, use the post type name
4208		- $content_type = sanitize_key($post_type);
4209		- break;
4210		- }
4211		-
4212		- // Use the centralized utility function with bot_id and content_type
	2056	+
	2057	+ // Use the centralized utility function for storage
4213	2058	$result = MxChat_Utils::submit_content_to_db(
4214		- $content,
4215		- $source_url,
	2059	+ $content,
	2060	+ $source_url,
4216	2061	$api_key,
4217		- $vector_id,
4218		- $bot_id,
4219		- $content_type
	2062	+ $vector_id
4220	2063	);
4221		-
	2064	+
4222	2065	if (is_wp_error($result)) {
4223		- MxChat_Admin::mxchat_log_debug('storage_error', 'Knowledge storage failed: ' . $result->get_error_message(), array('source_url' => $source_url));
4224	2066	wp_send_json_error('Storage failed: ' . $result->get_error_message());
4225	2067	exit;
4226	2068	}
4227		-
4228		- // Automatically apply role restriction based on tags
4229		- $this->apply_role_restriction_to_post($post_id, $source_url);
4230		-
	2069	+
	2070	+ // ================================
	2071	+ // UPDATE: Only update caches if Pinecone is enabled
	2072	+ // ================================
	2073	+
	2074	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2075	+ // Update vector ID cache for improved fetching
	2076	+ $this->mxchat_update_pinecone_vector_cache($vector_id);
	2077	+
	2078	+ // Update local processed content cache for immediate UI feedback
	2079	+ $pinecone_cache = get_option('mxchat_pinecone_processed_cache', array());
	2080	+ $pinecone_cache[$post_id] = array(
	2081	+ 'db_id' => $vector_id,
	2082	+ 'processed_date' => 'Just now',
	2083	+ 'url' => $source_url,
	2084	+ 'source' => 'pinecone',
	2085	+ 'timestamp' => current_time('timestamp')
	2086	+ );
	2087	+ update_option('mxchat_pinecone_processed_cache', $pinecone_cache);
	2088	+
	2089	+ // Also update the general processed content cache
	2090	+ $processed_cache = get_option('mxchat_processed_content_cache', array());
	2091	+ $processed_cache[$post_id] = array(
	2092	+ 'db_id' => $vector_id,
	2093	+ 'timestamp' => current_time('timestamp'),
	2094	+ 'url' => $source_url,
	2095	+ 'source' => 'pinecone'
	2096	+ );
	2097	+ update_option('mxchat_processed_content_cache', $processed_cache);
	2098	+ }
	2099	+
4231	2100	$operation_type = $is_update ? 'update' : 'new';
4232	2101
4233		- // Count ACF fields for debugging
4234		- $acf_field_count = count($acf_fields);
4235		-
4236	2102	// Success response with minimal data
4237	2103	wp_send_json_success(array(
4238	2104	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
4239	2105	'post_id' => $post_id,
		@@ -4238,95 +2104,27 @@
4238	2104	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
4239	2105	'post_id' => $post_id,
4240	2106	'title' => $post->post_title,
4241	2107	'operation_type' => $operation_type,
4242		- 'vector_id' => $vector_id,
4243		- 'acf_fields_found' => $acf_field_count,
4244		- 'pdf_extracted_count' => (int) $pdf_extracted_count,
4245		- 'content_preview' => substr($content, 0, 100) . '...',
4246		- 'bot_id' => $bot_id
	2108	+ 'vector_id' => $vector_id, // Include vector ID for debugging
	2109	+ 'cache_updated' => $use_pinecone // Indicate if cache was updated
4247	2110	));
4248	2111	exit;
4249	2112	}
4250	2113
4251		-private function apply_role_restriction_to_post($post_id, $source_url) {
4252		- // Get tag-role mappings
4253		- $mappings = get_option('mxchat_tag_role_mappings', array());
4254		-
4255		- if (empty($mappings)) {
4256		- return; // No mappings, leave as public
4257		- }
4258		-
4259		- // Get all tags for the post
4260		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
4261		-
4262		- if (empty($post_tags)) {
4263		- return; // No tags, leave as public
4264		- }
4265		-
4266		- // Determine the highest role restriction based on tags
4267		- $highest_role = 'public';
4268		- $role_hierarchy = array(
4269		- 'public' => 0,
4270		- 'logged_in' => 1,
4271		- 'subscriber' => 2,
4272		- 'contributor' => 3,
4273		- 'author' => 4,
4274		- 'editor' => 5,
4275		- 'administrator' => 6
4276		- );
4277		-
4278		- foreach ($post_tags as $tag_slug) {
4279		- if (isset($mappings[$tag_slug])) {
4280		- $role = $mappings[$tag_slug];
4281		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
4282		- $highest_role = $role;
4283		- }
4284		- }
4285		- }
4286		-
4287		- // If no restricted tags found, return (leave as public)
4288		- if ($highest_role === 'public') {
4289		- return;
4290		- }
4291		-
4292		- // Update the role restriction in the database
4293		- global $wpdb;
4294		-
4295		- // Check if using Pinecone
4296		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
4297		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
4298		-
4299		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
4300		- // Update Pinecone role restriction
4301		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
4302		- $vector_id = md5($source_url);
4303		-
4304		- $wpdb->replace(
4305		- $roles_table,
4306		- array(
4307		- 'vector_id' => $vector_id,
4308		- 'role_restriction' => $highest_role,
4309		- 'updated_at' => current_time('mysql')
4310		- ),
4311		- array('%s', '%s', '%s')
4312		- );
4313		- } else {
4314		- // Update WordPress DB
4315		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
4316		-
4317		- $wpdb->update(
4318		- $table_name,
4319		- array('role_restriction' => $highest_role),
4320		- array('source_url' => $source_url),
4321		- array('%s'),
4322		- array('%s')
4323		- );
4324		- }
4325		-}
4326	2114
	2115	+
	2116	+ /**
	2117	+ * Updates cache with new vector ID if absent
	2118	+ */
	2119	+ public function mxchat_update_pinecone_vector_cache($vector_id) {
	2120	+ $cached_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2121	+ if (!in_array($vector_id, $cached_ids)) {
	2122	+ $cached_ids[] = $vector_id;
	2123	+ update_option('mxchat_pinecone_vector_ids_cache', $cached_ids);
	2124	+ }
	2125	+ }
4327	2126	public function mxchat_get_public_post_types() {
4328		- // Get all public post types
4329	2127	$post_types = get_post_types(array('public' => true), 'objects');
4330	2128	$post_type_options = array();
4331	2129
4332	2130	foreach ($post_types as $post_type) {
		@@ -4332,45 +2130,51 @@
4332	2130	foreach ($post_types as $post_type) {
4333	2131	$post_type_options[$post_type->name] = $post_type->label;
4334	2132	}
4335	2133
4336		- // Also include common forum/community post types that might not be marked as public
4337		- $additional_types = array(
4338		- 'topic' => 'Forum Topics (bbPress)',
4339		- 'reply' => 'Forum Replies (bbPress)',
4340		- 'forum' => 'Forums (bbPress)',
4341		- 'wpforo_topic' => 'wpForo Topics',
4342		- 'wpforo_post' => 'wpForo Posts'
4343		- );
4344		-
4345		- foreach ($additional_types as $type_name => $type_label) {
4346		- if (post_type_exists($type_name) && !isset($post_type_options[$type_name])) {
4347		- $post_type_options[$type_name] = $type_label;
4348		- }
4349		- }
4350		-
4351	2134	return $post_type_options;
4352	2135	}
	2136	+public function mxchat_get_pinecone_processed_content($pinecone_options) {
	2137	+ //error_log('=== DEBUG: Starting mxchat_get_pinecone_processed_content ===');
	2138	+
	2139	+ // First check local cache for immediate updates
	2140	+ $cached_data = get_option('mxchat_pinecone_processed_cache', array());
	2141	+ //error_log('DEBUG: Found ' . count($cached_data) . ' items in local cache');
4353	2142
4354		-/**
4355		- * Retrieves processed content from Pinecone API
4356		- */
4357		-public function mxchat_get_pinecone_processed_content($pinecone_options) {
4358	2143	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4359	2144	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
	2145	+
	2146	+ //error_log('DEBUG: API key present: ' . (!empty($api_key) ? 'YES' : 'NO'));
	2147	+ //error_log('DEBUG: Host: ' . $host);
4360	2148
4361	2149	if (empty($api_key) || empty($host)) {
4362		- return array();
	2150	+ //error_log('DEBUG: Missing API credentials, returning cached data only');
	2151	+ return $cached_data;
4363	2152	}
4364	2153
4365	2154	$pinecone_data = array();
4366	2155
4367	2156	try {
4368		- // Always get fresh data from Pinecone
4369		- $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
	2157	+ // Method 1: Try to get vectors using cached vector IDs first
	2158	+ $cached_vector_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2159	+ //error_log('DEBUG: Found ' . count($cached_vector_ids) . ' cached vector IDs');
4370	2160
4371		- // Method 2: Final fallback - try stats endpoint (if available)
	2161	+ if (!empty($cached_vector_ids)) {
	2162	+ //error_log('DEBUG: Trying to fetch by cached vector IDs...');
	2163	+ $pinecone_data = $this->mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $cached_vector_ids);
	2164	+ //error_log('DEBUG: Fetch by IDs returned ' . count($pinecone_data) . ' items');
	2165	+ }
	2166	+
	2167	+ // Method 2: If no cached IDs or fetch failed, use scanning approach
4372	2168	if (empty($pinecone_data)) {
	2169	+ //error_log('DEBUG: Trying scanning approach...');
	2170	+ $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
	2171	+ //error_log('DEBUG: Scanning returned ' . count($pinecone_data) . ' items');
	2172	+ }
	2173	+
	2174	+ // Method 3: Final fallback - try stats endpoint
	2175	+ if (empty($pinecone_data)) {
	2176	+ //error_log('DEBUG: Trying stats endpoint...');
4373	2177	$stats_url = "https://{$host}/describe_index_stats";
4374	2178
4375	2179	$response = wp_remote_post($stats_url, array(
4376	2180	'headers' => array(
		@@ -4383,17 +2187,44 @@
4383	2187
4384	2188	if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
4385	2189	$body = wp_remote_retrieve_body($response);
4386	2190	$stats_data = json_decode($body, true);
	2191	+ //error_log('DEBUG: Pinecone stats: ' . print_r($stats_data, true));
	2192	+ } else {
	2193	+ if (is_wp_error($response)) {
	2194	+ //error_log('DEBUG: Stats endpoint error: ' . $response->get_error_message());
	2195	+ } else {
	2196	+ //error_log('DEBUG: Stats endpoint failed with code: ' . wp_remote_retrieve_response_code($response));
	2197	+ }
4387	2198	}
4388	2199	}
4389	2200
4390	2201	} catch (Exception $e) {
4391		- // Log error but return fresh data only
	2202	+ //error_log('DEBUG: Exception in get_pinecone_processed_content: ' . $e->getMessage());
4392	2203	}
4393	2204
4394		- return $pinecone_data;
	2205	+ // Merge cached data with Pinecone data
	2206	+ $merged_data = $pinecone_data;
	2207	+
	2208	+ foreach ($cached_data as $post_id => $cache_item) {
	2209	+ $cache_timestamp = $cache_item['timestamp'] ?? 0;
	2210	+ $time_diff = current_time('timestamp') - $cache_timestamp;
	2211	+
	2212	+ if ($time_diff < 300) { // 5 minutes = 300 seconds
	2213	+ $merged_data[$post_id] = $cache_item;
	2214	+ } else {
	2215	+ if (!isset($merged_data[$post_id])) {
	2216	+ $merged_data[$post_id] = $cache_item;
	2217	+ }
	2218	+ }
	2219	+ }
	2220	+
	2221	+ //error_log('DEBUG: Final merged data count: ' . count($merged_data));
	2222	+ //error_log('=== DEBUG: End mxchat_get_pinecone_processed_content ===');
	2223	+
	2224	+ return $merged_data;
4395	2225	}
	2226	+
4396	2227	public function mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $vector_ids) {
4397	2228	//error_log('=== DEBUG: Starting mxchat_fetch_pinecone_vectors_by_ids ===');
4398	2229
4399	2230	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
		@@ -4485,46 +2316,16 @@
4485	2316	return array();
4486	2317	}
4487	2318	}
4488	2319
4489		-/**
4490		- * Get embedding dimensions based on the selected model.
4491		- */
4492		-private function mxchat_get_embedding_dimensions() {
4493		- $options = get_option('mxchat_options', array());
4494		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4495		-
4496		- $model_dimensions = array(
4497		- 'text-embedding-ada-002' => 1536,
4498		- 'text-embedding-3-small' => 1536,
4499		- 'text-embedding-3-large' => 3072,
4500		- 'voyage-2' => 1024,
4501		- 'voyage-large-2' => 1536,
4502		- 'voyage-3-large' => 2048,
4503		- 'gemini-embedding-001' => 1536,
4504		- );
4505		-
4506		- if (strpos($selected_model, 'voyage-3-large') === 0) {
4507		- $custom_dimensions = $options['voyage_output_dimension'] ?? 2048;
4508		- return intval($custom_dimensions);
4509		- }
4510		-
4511		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4512		- $custom_dimensions = $options['gemini_output_dimension'] ?? 1536;
4513		- return intval($custom_dimensions);
4514		- }
4515		-
4516		- return $model_dimensions[$selected_model] ?? 1536;
4517		-}
4518		-
4519		-/**
4520		- * Scan Pinecone for processed content
4521		- */
4522	2320	public function mxchat_scan_pinecone_for_processed_content($pinecone_options) {
	2321	+ //error_log('=== DEBUG: Starting mxchat_scan_pinecone_for_processed_content ===');
	2322	+
4523	2323	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4524	2324	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
4525	2325
4526	2326	if (empty($api_key) || empty($host)) {
	2327	+ //error_log('DEBUG: Missing API credentials for scanning');
4527	2328	return array();
4528	2329	}
4529	2330
4530	2331	try {
		@@ -4531,18 +2332,17 @@
4531	2332	// Use multiple random vectors to get better coverage
4532	2333	$all_matches = array();
4533	2334	$seen_ids = array();
4534	2335
4535		- // Get correct dimensions for the configured embedding model
4536		- $dimensions = $this->mxchat_get_embedding_dimensions();
4537		-
4538	2336	// Try 3 different random vectors to get better coverage
4539	2337	for ($i = 0; $i < 3; $i++) {
	2338	+ //error_log('DEBUG: Scanning attempt ' . ($i + 1) . '/3');
	2339	+
4540	2340	$query_url = "https://{$host}/query";
4541	2341
4542	2342	// Generate a random unit vector instead of zeros
4543	2343	$random_vector = array();
4544		- for ($j = 0; $j < $dimensions; $j++) {
	2344	+ for ($j = 0; $j < 1536; $j++) {
4545	2345	$random_vector[] = (rand(-1000, 1000) / 1000.0);
4546	2346	}
4547	2347
4548	2348	// Normalize the vector to unit length
		@@ -4567,14 +2367,18 @@
4567	2367	'timeout' => 30
4568	2368	));
4569	2369
4570	2370	if (is_wp_error($response)) {
	2371	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' WP error: ' . $response->get_error_message());
4571	2372	continue;
4572	2373	}
4573	2374
4574	2375	$response_code = wp_remote_retrieve_response_code($response);
	2376	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' response code: ' . $response_code);
4575	2377
4576	2378	if ($response_code !== 200) {
	2379	+ $error_body = wp_remote_retrieve_body($response);
	2380	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' failed with body: ' . substr($error_body, 0, 500));
4577	2381	continue;
4578	2382	}
4579	2383
4580	2384	$body = wp_remote_retrieve_body($response);
		@@ -4580,8 +2384,9 @@
4580	2384	$body = wp_remote_retrieve_body($response);
4581	2385	$data = json_decode($body, true);
4582	2386
4583	2387	if (isset($data['matches'])) {
	2388	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' returned ' . count($data['matches']) . ' matches');
4584	2389	foreach ($data['matches'] as $match) {
4585	2390	$match_id = $match['id'] ?? '';
4586	2391	if (!empty($match_id) && !isset($seen_ids[$match_id])) {
4587	2392	$all_matches[] = $match;
		@@ -4587,14 +2392,18 @@
4587	2392	$all_matches[] = $match;
4588	2393	$seen_ids[$match_id] = true;
4589	2394	}
4590	2395	}
	2396	+ } else {
	2397	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' - no matches key in response');
4591	2398	}
4592	2399	}
4593	2400
4594		- // Convert matches to processed data format, grouping by URL to count chunks
	2401	+ //error_log('DEBUG: Total unique matches found: ' . count($all_matches));
	2402	+
	2403	+ // Convert matches to processed data format
4595	2404	$processed_data = array();
4596		- $url_chunk_counts = array();
	2405	+ $vector_ids_for_cache = array();
4597	2406
4598	2407	foreach ($all_matches as $match) {
4599	2408	$metadata = $match['metadata'] ?? array();
4600	2409	$source_url = $metadata['source_url'] ?? '';
		@@ -4602,14 +2411,8 @@
4602	2411
4603	2412	if (!empty($source_url) && !empty($match_id)) {
4604	2413	$post_id = url_to_postid($source_url);
4605	2414	if ($post_id) {
4606		- // Count chunks per post_id
4607		- if (!isset($url_chunk_counts[$post_id])) {
4608		- $url_chunk_counts[$post_id] = 0;
4609		- }
4610		- $url_chunk_counts[$post_id]++;
4611		-
4612	2415	$created_at = $metadata['created_at'] ?? '';
4613	2416	$processed_date = 'Recently';
4614	2417
4615	2418	if (!empty($created_at)) {
		@@ -4618,322 +2421,243 @@
4618	2421	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
4619	2422	}
4620	2423	}
4621	2424
4622		- // Only store if not already set, or update with newer timestamp
4623		- if (!isset($processed_data[$post_id]) ||
4624		- ($timestamp ?? 0) > ($processed_data[$post_id]['timestamp'] ?? 0)) {
4625		- $processed_data[$post_id] = array(
4626		- 'db_id' => $match_id,
4627		- 'processed_date' => $processed_date,
4628		- 'url' => $source_url,
4629		- 'source' => 'pinecone',
4630		- 'timestamp' => $timestamp ?? current_time('timestamp')
4631		- );
4632		- }
	2425	+ $processed_data[$post_id] = array(
	2426	+ 'db_id' => $match_id,
	2427	+ 'processed_date' => $processed_date,
	2428	+ 'url' => $source_url,
	2429	+ 'source' => 'pinecone',
	2430	+ 'timestamp' => $timestamp ?? current_time('timestamp')
	2431	+ );
	2432	+
	2433	+ $vector_ids_for_cache[] = $match_id;
4633	2434	}
4634	2435	}
4635	2436	}
4636	2437
4637		- // Add chunk counts to processed data
4638		- foreach ($url_chunk_counts as $post_id => $chunk_count) {
4639		- if (isset($processed_data[$post_id])) {
4640		- $processed_data[$post_id]['chunk_count'] = $chunk_count;
4641		- }
	2438	+ // Update the vector IDs cache for future use
	2439	+ if (!empty($vector_ids_for_cache)) {
	2440	+ update_option('mxchat_pinecone_vector_ids_cache', $vector_ids_for_cache);
	2441	+ //error_log('DEBUG: Updated vector IDs cache with ' . count($vector_ids_for_cache) . ' IDs');
4642	2442	}
4643	2443
	2444	+ //error_log('DEBUG: Returning ' . count($processed_data) . ' processed items from scanning');
4644	2445	return $processed_data;
4645	2446
4646	2447	} catch (Exception $e) {
	2448	+ //error_log('DEBUG: Exception in scan_pinecone_for_processed_content: ' . $e->getMessage());
4647	2449	return array();
4648	2450	}
4649	2451	}
4650		-/**
4651		- * Generate embeddings from input text for MXChat with bot support
4652		- */
4653		-private function mxchat_generate_embedding($text, $bot_id = 'default') {
4654		- // Enable detailed logging for debugging
4655		- //error_log('[MXCHAT-EMBED] Starting embedding generation for bot: ' . $bot_id . '. Text length: ' . strlen($text) . ' bytes');
4656		- //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4657	2452
4658		- // Get bot-specific options
4659		- $bot_options = $this->get_bot_options($bot_id);
4660		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	2453	+ /**
	2454	+ * Generates embeddings from input text for MXChat
	2455	+ */
	2456	+ private function mxchat_generate_embedding($text) {
	2457	+ // Enable detailed logging for debugging
	2458	+ //error_log('[MXCHAT-EMBED] Starting embedding generation. Text length: ' . strlen($text) . ' bytes');
	2459	+ //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4661	2460
4662		- // Opt-in: when the custom provider is selected for embeddings, index through
4663		- // the same custom endpoint the query path uses so stored vectors and query
4664		- // vectors share a model. Returns the vector array on success, or an error
4665		- // string on failure (this function's existing failure contract).
4666		- if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
4667		- if (!class_exists('MxChat_Utils')) {
4668		- require_once dirname(__FILE__) . '/../includes/class-mxchat-utils.php';
4669		- }
4670		- return MxChat_Utils::generate_embedding_custom($text, $options);
4671		- }
	2461	+ $options = get_option('mxchat_options');
	2462	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	2463	+ //error_log('[MXCHAT-EMBED] Selected embedding model: ' . $selected_model);
4672	2464
4673		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4674		- //error_log('[MXCHAT-EMBED] Selected embedding model for bot ' . $bot_id . ': ' . $selected_model);
	2465	+ // Determine provider and endpoint
	2466	+ if (strpos($selected_model, 'voyage') === 0) {
	2467	+ $api_key = $options['voyage_api_key'] ?? '';
	2468	+ $endpoint = 'https://api.voyageai.com/v1/embeddings';
	2469	+ $provider_name = 'Voyage AI';
	2470	+ //error_log('[MXCHAT-EMBED] Using Voyage AI API');
	2471	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	2472	+ $api_key = $options['gemini_api_key'] ?? '';
	2473	+ $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
	2474	+ $provider_name = 'Google Gemini';
	2475	+ //error_log('[MXCHAT-EMBED] Using Google Gemini API');
	2476	+ } else {
	2477	+ $api_key = $options['api_key'] ?? '';
	2478	+ $endpoint = 'https://api.openai.com/v1/embeddings';
	2479	+ $provider_name = 'OpenAI';
	2480	+ //error_log('[MXCHAT-EMBED] Using OpenAI API');
	2481	+ }
4675	2482
4676		- // Determine provider and endpoint
4677		- if (strpos($selected_model, 'voyage') === 0) {
4678		- $api_key = $options['voyage_api_key'] ?? '';
4679		- $endpoint = 'https://api.voyageai.com/v1/embeddings';
4680		- $provider_name = 'Voyage AI';
4681		- //error_log('[MXCHAT-EMBED] Using Voyage AI API for bot ' . $bot_id);
4682		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
4683		- $api_key = $options['gemini_api_key'] ?? '';
4684		- $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
4685		- $provider_name = 'Google Gemini';
4686		- //error_log('[MXCHAT-EMBED] Using Google Gemini API for bot ' . $bot_id);
4687		- } else {
4688		- $api_key = $options['api_key'] ?? '';
4689		- $endpoint = 'https://api.openai.com/v1/embeddings';
4690		- $provider_name = 'OpenAI';
4691		- //error_log('[MXCHAT-EMBED] Using OpenAI API for bot ' . $bot_id);
4692		- }
	2483	+ //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
4693	2484
4694		- //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
	2485	+ if (empty($api_key)) {
	2486	+ $error_message = sprintf('Missing %s API key. Please configure your API key in the MxChat settings.', $provider_name);
	2487	+ //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
	2488	+ return $error_message;
	2489	+ }
4695	2490
4696		- if (empty($api_key)) {
4697		- $error_message = sprintf('Missing %s API key for bot %s. Please configure your API key in the bot settings.', $provider_name, $bot_id);
4698		- //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
4699		- return $error_message;
4700		- }
	2491	+ // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
	2492	+ $estimated_tokens = ceil(str_word_count($text) / 0.75);
	2493	+ //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
4701	2494
4702		- // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
4703		- $estimated_tokens = ceil(str_word_count($text) / 0.75);
4704		- //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
	2495	+ if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
	2496	+ //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
	2497	+ // Consider truncating text here
	2498	+ }
4705	2499
4706		- if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
4707		- //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
4708		- // Consider truncating text here
4709		- }
	2500	+ // Prepare request body based on provider
	2501	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2502	+ // Gemini API format
	2503	+ $request_body = array(
	2504	+ 'model' => 'models/' . $selected_model,
	2505	+ 'content' => array(
	2506	+ 'parts' => array(
	2507	+ array('text' => $text)
	2508	+ )
	2509	+ )
	2510	+ );
4710	2511
4711		- // Prepare request body based on provider
4712		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4713		- // Gemini API format
4714		- $request_body = array(
4715		- 'model' => 'models/' . $selected_model,
4716		- 'content' => array(
4717		- 'parts' => array(
4718		- array('text' => $text)
4719		- )
4720		- )
4721		- );
	2512	+ // Set output dimensionality to 1536 for consistency with other models
	2513	+ $request_body['outputDimensionality'] = 1536;
	2514	+ } else {
	2515	+ // OpenAI/Voyage API format
	2516	+ $request_body = array(
	2517	+ 'model' => $selected_model,
	2518	+ 'input' => $text
	2519	+ );
4722	2520
4723		- // Set output dimensionality to 1536 for consistency with other models
4724		- $request_body['outputDimensionality'] = 1536;
4725		- } else {
4726		- // OpenAI/Voyage API format
4727		- $request_body = array(
4728		- 'model' => $selected_model,
4729		- 'input' => $text
4730		- );
	2521	+ // Add output_dimension for voyage-3-large model
	2522	+ if ($selected_model === 'voyage-3-large') {
	2523	+ $request_body['output_dimension'] = 2048;
	2524	+ }
	2525	+ }
4731	2526
4732		- // Add output_dimension for voyage-3-large model
4733		- if ($selected_model === 'voyage-3-large') {
4734		- $request_body['output_dimension'] = 2048;
4735		- }
4736		- }
	2527	+ //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
4737	2528
4738		- //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
	2529	+ // Prepare headers based on provider
	2530	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2531	+ // Gemini uses API key as query parameter
	2532	+ $endpoint .= '?key=' . $api_key;
	2533	+ $headers = array(
	2534	+ 'Content-Type' => 'application/json'
	2535	+ );
	2536	+ } else {
	2537	+ // OpenAI/Voyage use Bearer token
	2538	+ $headers = array(
	2539	+ 'Authorization' => 'Bearer ' . $api_key,
	2540	+ 'Content-Type' => 'application/json'
	2541	+ );
	2542	+ }
4739	2543
4740		- // Prepare headers based on provider
4741		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4742		- // Gemini uses API key as query parameter
4743		- $endpoint .= '?key=' . $api_key;
4744		- $headers = array(
4745		- 'Content-Type' => 'application/json'
4746		- );
4747		- } else {
4748		- // OpenAI/Voyage use Bearer token
4749		- $headers = array(
4750		- 'Authorization' => 'Bearer ' . $api_key,
4751		- 'Content-Type' => 'application/json'
4752		- );
4753		- }
	2544	+ // Make API request
	2545	+ //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
	2546	+ $response = wp_remote_post($endpoint, array(
	2547	+ 'body' => wp_json_encode($request_body),
	2548	+ 'headers' => $headers,
	2549	+ 'timeout' => 60 // Increased timeout for large inputs
	2550	+ ));
4754	2551
4755		- // Make API request
4756		- //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
4757		- $response = wp_remote_post($endpoint, array(
4758		- 'body' => wp_json_encode($request_body),
4759		- 'headers' => $headers,
4760		- 'timeout' => 60 // Increased timeout for large inputs
4761		- ));
	2552	+ // Handle wp_remote_post errors
	2553	+ if (is_wp_error($response)) {
	2554	+ $error_message = $response->get_error_message();
	2555	+ //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
	2556	+ return 'Connection error: ' . $error_message;
	2557	+ }
4762	2558
4763		- // Handle wp_remote_post errors
4764		- if (is_wp_error($response)) {
4765		- $error_message = $response->get_error_message();
4766		- //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
4767		- return 'Connection error: ' . $error_message;
4768		- }
	2559	+ // Get and check HTTP response code
	2560	+ $http_code = wp_remote_retrieve_response_code($response);
	2561	+ //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
4769	2562
4770		- // Get and check HTTP response code
4771		- $http_code = wp_remote_retrieve_response_code($response);
4772		- //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
	2563	+ if ($http_code !== 200) {
	2564	+ $error_body = wp_remote_retrieve_body($response);
	2565	+ //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
4773	2566
4774		- if ($http_code !== 200) {
4775		- $error_body = wp_remote_retrieve_body($response);
4776		- //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
	2567	+ // Try to parse error for more details
	2568	+ $error_json = json_decode($error_body, true);
	2569	+ if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
	2570	+ $error_type = $error_json['error']['type'] ?? 'unknown';
	2571	+ $error_message = $error_json['error']['message'] ?? 'No message';
	2572	+ //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
	2573	+ //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
4777	2574
4778		- // Try to parse error for more details
4779		- $error_json = json_decode($error_body, true);
4780		- if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
4781		- $error_type = $error_json['error']['type'] ?? 'unknown';
4782		- $error_message = $error_json['error']['message'] ?? 'No message';
4783		- //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
4784		- //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
	2575	+ // Customize error message for common API errors
	2576	+ if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
	2577	+ $error_message = sprintf('Invalid %s API key. Please check your API key in the MxChat settings.', $provider_name);
	2578	+ } elseif ($error_type === 'authentication_error') {
	2579	+ $error_message = sprintf('%s authentication failed. Please verify your API key in the MxChat settings.', $provider_name);
	2580	+ }
4785	2581
4786		- // Customize error message for common API errors
4787		- if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
4788		- $error_message = sprintf('Invalid %s API key for bot %s. Please check your API key in the bot settings.', $provider_name, $bot_id);
4789		- } elseif ($error_type === 'authentication_error') {
4790		- $error_message = sprintf('%s authentication failed for bot %s. Please verify your API key in the bot settings.', $provider_name, $bot_id);
4791		- }
	2582	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2583	+ return $error_message;
	2584	+ }
4792	2585
4793		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4794		- return $error_message;
4795		- }
	2586	+ $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding", $http_code);
	2587	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2588	+ return $error_message;
	2589	+ }
4796	2590
4797		- $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding for bot %s", $http_code, $bot_id);
4798		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4799		- return $error_message;
4800		- }
	2591	+ // Parse response body
	2592	+ $response_body = wp_remote_retrieve_body($response);
	2593	+ //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
4801	2594
4802		- // Parse response body
4803		- $response_body = wp_remote_retrieve_body($response);
4804		- //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
	2595	+ $response_data = json_decode($response_body, true);
4805	2596
4806		- $response_data = json_decode($response_body, true);
	2597	+ if (json_last_error() !== JSON_ERROR_NONE) {
	2598	+ $error = json_last_error_msg();
	2599	+ //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
	2600	+ //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
	2601	+ return "Failed to parse API response: $error";
	2602	+ }
4807	2603
4808		- if (json_last_error() !== JSON_ERROR_NONE) {
4809		- $error = json_last_error_msg();
4810		- //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
4811		- //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
4812		- return "Failed to parse API response: $error";
4813		- }
	2604	+ // Handle different response formats based on provider
	2605	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2606	+ // Gemini API response format
	2607	+ if (isset($response_data['embedding']['values'])) {
	2608	+ $embedding_dimensions = count($response_data['embedding']['values']);
	2609	+ //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
4814	2610
4815		- // Handle different response formats based on provider
4816		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4817		- // Gemini API response format
4818		- if (isset($response_data['embedding']['values'])) {
4819		- $embedding_dimensions = count($response_data['embedding']['values']);
4820		- //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
	2611	+ // Check if embedding dimensions are as expected (should be 1536)
	2612	+ if ($embedding_dimensions !== 1536) {
	2613	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
	2614	+ }
4821	2615
4822		- // Check if embedding dimensions are as expected (should be 1536)
4823		- if ($embedding_dimensions !== 1536) {
4824		- //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
4825		- }
	2616	+ return $response_data['embedding']['values'];
	2617	+ } else {
	2618	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
	2619	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4826	2620
4827		- return $response_data['embedding']['values'];
4828		- } else {
4829		- //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
4830		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2621	+ if (isset($response_data['error'])) {
	2622	+ $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
	2623	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2624	+ return $error_message;
	2625	+ }
4831	2626
4832		- if (isset($response_data['error'])) {
4833		- $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
4834		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4835		- return $error_message;
4836		- }
	2627	+ $error_message = "Invalid Gemini API response format: No embedding found";
	2628	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2629	+ return $error_message;
	2630	+ }
	2631	+ } else {
	2632	+ // OpenAI/Voyage API response format
	2633	+ if (isset($response_data['data'][0]['embedding'])) {
	2634	+ $embedding_dimensions = count($response_data['data'][0]['embedding']);
	2635	+ //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
4837	2636
4838		- $error_message = "Invalid Gemini API response format: No embedding found";
4839		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4840		- return $error_message;
4841		- }
4842		- } else {
4843		- // OpenAI/Voyage API response format
4844		- if (isset($response_data['data'][0]['embedding'])) {
4845		- $embedding_dimensions = count($response_data['data'][0]['embedding']);
4846		- //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
	2637	+ // Check if embedding dimensions are as expected
	2638	+ if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
	2639	+ ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
	2640	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
	2641	+ }
4847	2642
4848		- // Check if embedding dimensions are as expected
4849		- if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
4850		- ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
4851		- //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
4852		- }
	2643	+ return $response_data['data'][0]['embedding'];
	2644	+ } else {
	2645	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
	2646	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4853	2647
4854		- return $response_data['data'][0]['embedding'];
4855		- } else {
4856		- //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
4857		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2648	+ if (isset($response_data['error'])) {
	2649	+ $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
	2650	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2651	+ return $error_message;
	2652	+ }
4858	2653
4859		- if (isset($response_data['error'])) {
4860		- $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
4861		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4862		- return $error_message;
4863		- }
4864		-
4865		- $error_message = "Invalid API response format: No embedding found";
4866		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4867		- return $error_message;
4868		- }
4869		- }
4870		-}
4871		-
4872		-/**
4873		- * Get bot-specific options for multi-bot functionality
4874		- * Falls back to default options if bot_id is 'default' or multi-bot add-on is not active
4875		- */
4876		-private function get_bot_options($bot_id = 'default') {
4877		- //error_log("MXCHAT DEBUG: get_bot_options called for bot: " . $bot_id);
4878		-
4879		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4880		- //error_log("MXCHAT DEBUG: Using default options (no multi-bot or bot is 'default')");
4881		- return array();
4882		- }
4883		-
4884		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
4885		-
4886		- if (!empty($bot_options)) {
4887		- //error_log("MXCHAT DEBUG: Got bot-specific options from filter");
4888		- if (isset($bot_options['similarity_threshold'])) {
4889		- //error_log(" - similarity_threshold: " . $bot_options['similarity_threshold']);
4890		- }
4891		- }
4892		-
4893		- return is_array($bot_options) ? $bot_options : array();
4894		-}
4895		-
4896		-/**
4897		- * Get bot-specific Pinecone configuration
4898		- * Used in the knowledge retrieval functions
4899		- */
4900		-// Also add debugging to your get_bot_pinecone_config function
4901		-private function get_bot_pinecone_config($bot_id = 'default') {
4902		- //error_log("MXCHAT DEBUG: get_bot_pinecone_config called for bot: " . $bot_id);
4903		-
4904		- // If default bot or multi-bot add-on not active, use default Pinecone config
4905		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4906		- //error_log("MXCHAT DEBUG: Using default Pinecone config (no multi-bot or bot is 'default')");
4907		- $addon_options = get_option('mxchat_pinecone_addon_options', array());
4908		- $config = array(
4909		- 'use_pinecone' => (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1'),
4910		- 'api_key' => $addon_options['mxchat_pinecone_api_key'] ?? '',
4911		- 'host' => $addon_options['mxchat_pinecone_host'] ?? '',
4912		- 'namespace' => $addon_options['mxchat_pinecone_namespace'] ?? ''
4913		- );
4914		- //error_log("MXCHAT DEBUG: Default config - use_pinecone: " . ($config['use_pinecone'] ? 'true' : 'false'));
4915		- return $config;
4916		- }
4917		-
4918		- //error_log("MXCHAT DEBUG: Calling filter 'mxchat_get_bot_pinecone_config' for bot: " . $bot_id);
4919		-
4920		- // Hook for multi-bot add-on to provide bot-specific Pinecone config
4921		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
4922		-
4923		- if (!empty($bot_pinecone_config)) {
4924		- //error_log("MXCHAT DEBUG: Got bot-specific config from filter");
4925		- //error_log(" - use_pinecone: " . (isset($bot_pinecone_config['use_pinecone']) ? ($bot_pinecone_config['use_pinecone'] ? 'true' : 'false') : 'not set'));
4926		- //error_log(" - host: " . ($bot_pinecone_config['host'] ?? 'not set'));
4927		- //error_log(" - namespace: " . ($bot_pinecone_config['namespace'] ?? 'not set'));
4928		- } else {
4929		- //error_log("MXCHAT DEBUG: Filter returned empty config!");
4930		- }
4931		-
4932		- return is_array($bot_pinecone_config) ? $bot_pinecone_config : array();
4933		-}
4934		-
4935		-
	2654	+ $error_message = "Invalid API response format: No embedding found";
	2655	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2656	+ return $error_message;
	2657	+ }
	2658	+ }
	2659	+ }
4936	2660	public function mxchat_ajax_dismiss_completed_status() {
4937	2661	try {
4938	2662	// Verify the request
4939	2663	check_ajax_referer('mxchat_status_nonce', 'nonce');
		@@ -5296,564 +3020,18 @@
5296	3020
5297	3021	return $html;
5298	3022	}
5299	3023
5300		-/**
5301		- * Get all ACF fields for a specific post, excluding any fields the user has disabled
5302		- */
5303		-public function mxchat_get_acf_fields_for_post($post_id) {
5304		- if (!function_exists('get_fields')) {
5305		- return array();
5306		- }
5307		-
5308		- $fields = get_fields($post_id);
5309		- if (!$fields || !is_array($fields)) {
5310		- return array();
5311		- }
5312		-
5313		- // Get excluded fields from settings
5314		- $excluded_fields = get_option('mxchat_acf_excluded_fields', array());
5315		- if (!empty($excluded_fields) && is_array($excluded_fields)) {
5316		- foreach ($excluded_fields as $excluded_field) {
5317		- if (isset($fields[$excluded_field])) {
5318		- unset($fields[$excluded_field]);
5319		- }
5320		- }
5321		- }
5322		-
5323		- return $fields;
5324		-}
5325		-
5326		-/**
5327		- * Get all registered ACF field groups and their fields for the settings UI
5328		- */
5329		-public function mxchat_get_all_acf_fields() {
5330		- if (!function_exists('acf_get_field_groups') || !function_exists('acf_get_fields')) {
5331		- return array();
5332		- }
5333		-
5334		- $all_fields = array();
5335		- $field_groups = acf_get_field_groups();
5336		-
5337		- if (!empty($field_groups)) {
5338		- foreach ($field_groups as $group) {
5339		- $group_fields = acf_get_fields($group['key']);
5340		- if (!empty($group_fields)) {
5341		- $all_fields[$group['title']] = array();
5342		- foreach ($group_fields as $field) {
5343		- $all_fields[$group['title']][] = array(
5344		- 'name' => $field['name'],
5345		- 'label' => $field['label'],
5346		- 'type' => $field['type']
5347		- );
5348		- }
5349		- }
5350		- }
5351		- }
5352		-
5353		- return $all_fields;
5354		-}
5355		-
5356		-/**
5357		- * Get whitelisted custom post meta for a given post
5358		- * This allows non-ACF meta fields (like OptionTree, theme meta boxes) to be included in embeddings
5359		- */
5360		-public function mxchat_get_whitelisted_post_meta($post_id) {
5361		- $whitelist = get_option('mxchat_custom_meta_whitelist', '');
5362		-
5363		- if (empty($whitelist)) {
5364		- return array();
5365		- }
5366		-
5367		- // Parse the whitelist - one meta key per line
5368		- $meta_keys = array_filter(array_map('trim', explode("\n", $whitelist)));
5369		-
5370		- if (empty($meta_keys)) {
5371		- return array();
5372		- }
5373		-
5374		- $result = array();
5375		-
5376		- foreach ($meta_keys as $key) {
5377		- // Skip empty keys
5378		- if (empty($key)) {
5379		- continue;
5380		- }
5381		-
5382		- $value = get_post_meta($post_id, $key, true);
5383		-
5384		- // Only include non-empty string values
5385		- if (!empty($value) && is_string($value)) {
5386		- $result[$key] = $value;
5387		- } elseif (!empty($value) && is_array($value)) {
5388		- // Handle array values by joining them
5389		- $flat_value = $this->mxchat_flatten_meta_array($value);
5390		- if (!empty($flat_value)) {
5391		- $result[$key] = $flat_value;
5392		- }
5393		- }
5394		- }
5395		-
5396		- return $result;
5397		-}
5398		-
5399		-/**
5400		- * Flatten array meta values into a readable string
5401		- */
5402		-private function mxchat_flatten_meta_array($array, $depth = 0) {
5403		- if ($depth > 3) {
5404		- return ''; // Prevent infinite recursion
5405		- }
5406		-
5407		- $parts = array();
5408		-
5409		- foreach ($array as $key => $value) {
5410		- if (is_string($value) && !empty($value)) {
5411		- $parts[] = $value;
5412		- } elseif (is_array($value)) {
5413		- $nested = $this->mxchat_flatten_meta_array($value, $depth + 1);
5414		- if (!empty($nested)) {
5415		- $parts[] = $nested;
5416		- }
5417		- }
5418		- }
5419		-
5420		- return implode(', ', $parts);
5421		-}
5422		-
5423		-/**
5424		- * Format ACF field values for content extraction
5425		- */
5426		-public function mxchat_format_acf_field_value($value, $field_name = '', $post_id = 0) {
5427		- if (empty($value)) {
5428		- return '';
5429		- }
5430		-
5431		- // Handle WP_Post objects first (THIS IS THE KEY FIX)
5432		- if ($value instanceof WP_Post) {
5433		- return $value->post_title ?: '';
5434		- }
5435		-
5436		- // Handle other WP objects
5437		- if (is_object($value)) {
5438		- if (isset($value->post_title)) {
5439		- return $value->post_title;
5440		- } elseif (isset($value->display_name)) {
5441		- return $value->display_name;
5442		- } elseif (isset($value->name)) {
5443		- return $value->name;
5444		- } elseif (method_exists($value, '__toString')) {
5445		- try {
5446		- return (string) $value;
5447		- } catch (Exception $e) {
5448		- return '';
5449		- }
5450		- }
5451		- // For any other objects, return empty string
5452		- return '';
5453		- }
5454		-
5455		- // Handle different ACF field types
5456		- if (is_array($value)) {
5457		- // Check if it's an image/file field
5458		- if (isset($value['url'])) {
5459		- // Image field - return alt text, title, or caption
5460		- if (!empty($value['alt'])) {
5461		- return $value['alt'];
5462		- } elseif (!empty($value['title'])) {
5463		- return $value['title'];
5464		- } elseif (!empty($value['caption'])) {
5465		- return $value['caption'];
5466		- } else {
5467		- return ''; // Don't include just the URL
5468		- }
5469		- }
5470		-
5471		- // Check if it's a post object or relationship field
5472		- if (isset($value['post_title'])) {
5473		- return $value['post_title'];
5474		- }
5475		-
5476		- // Check if it's a user field
5477		- if (isset($value['display_name'])) {
5478		- return $value['display_name'];
5479		- }
5480		-
5481		- // Check if it's a taxonomy term
5482		- if (isset($value['name']) && isset($value['taxonomy'])) {
5483		- return $value['name'];
5484		- }
5485		-
5486		- // Check if it's a select field with label
5487		- if (isset($value['label'])) {
5488		- return $value['label'];
5489		- }
5490		-
5491		- // Check for repeater field or flexible content
5492		- if (is_numeric(key($value))) {
5493		- $sub_values = array();
5494		- foreach ($value as $sub_item) {
5495		- if (is_array($sub_item)) {
5496		- // For repeater/flexible content, extract text values
5497		- $sub_text = $this->mxchat_extract_text_from_acf_array($sub_item);
5498		- if (!empty($sub_text)) {
5499		- $sub_values[] = $sub_text;
5500		- }
5501		- } elseif ($sub_item instanceof WP_Post) {
5502		- // Handle WP_Post objects in arrays
5503		- $sub_values[] = $sub_item->post_title ?: '';
5504		- } else {
5505		- $sub_values[] = (string) $sub_item;
5506		- }
5507		- }
5508		- return implode(', ', array_filter($sub_values));
5509		- }
5510		-
5511		- // For other arrays, try to extract meaningful text
5512		- $text_values = array();
5513		- foreach ($value as $key => $val) {
5514		- if (is_string($val) && !empty(trim($val))) {
5515		- $text_values[] = trim($val);
5516		- } elseif ($val instanceof WP_Post) {
5517		- // Handle WP_Post objects in associative arrays
5518		- $text_values[] = $val->post_title ?: '';
5519		- } elseif (is_array($val) && isset($val['post_title'])) {
5520		- $text_values[] = $val['post_title'];
5521		- } elseif (is_array($val) && isset($val['name'])) {
5522		- $text_values[] = $val['name'];
5523		- }
5524		- }
5525		-
5526		- return implode(', ', array_filter($text_values));
5527		- }
5528		-
5529		- // Handle boolean values
5530		- if (is_bool($value)) {
5531		- return $value ? 'Yes' : 'No';
5532		- }
5533		-
5534		- // Handle numeric values
5535		- if (is_numeric($value)) {
5536		- return (string) $value;
5537		- }
5538		-
5539		- // Handle string values
5540		- if (is_string($value)) {
5541		- return trim($value);
5542		- }
5543		-
5544		- // For anything else that we can't handle, return empty string
5545		- // This prevents the "Object could not be converted to string" error
5546		- return '';
5547		-}
5548		-
5549		-/**
5550		- * Extract text from complex ACF array structures
5551		- */
5552		-private function mxchat_extract_text_from_acf_array($array) {
5553		- if (!is_array($array)) {
5554		- return '';
5555		- }
5556		-
5557		- $text_parts = array();
5558		-
5559		- foreach ($array as $key => $value) {
5560		- if (is_string($value) && !empty(trim($value))) {
5561		- // Skip keys that are likely to be IDs or technical values
5562		- if (!is_numeric($value) || strlen($value) > 10) {
5563		- $text_parts[] = trim($value);
5564		- }
5565		- } elseif ($value instanceof WP_Post) {
5566		- // Handle WP_Post objects
5567		- $text_parts[] = $value->post_title ?: '';
5568		- } elseif (is_array($value)) {
5569		- if (isset($value['post_title'])) {
5570		- $text_parts[] = $value['post_title'];
5571		- } elseif (isset($value['name'])) {
5572		- $text_parts[] = $value['name'];
5573		- } elseif (isset($value['label'])) {
5574		- $text_parts[] = $value['label'];
5575		- }
5576		- } elseif (is_object($value)) {
5577		- // Handle other objects safely
5578		- if (isset($value->post_title)) {
5579		- $text_parts[] = $value->post_title;
5580		- } elseif (isset($value->name)) {
5581		- $text_parts[] = $value->name;
5582		- } elseif (isset($value->display_name)) {
5583		- $text_parts[] = $value->display_name;
5584		- }
5585		- }
5586		- }
5587		-
5588		- return implode(', ', array_filter($text_parts));
5589		-}
5590		-
5591		-/**
5592		- * Walk an ACF field value tree and collect attachment IDs for any value that
5593		- * resolves to a PDF in the WordPress media library. Handles the three shapes
5594		- * ACF returns for File/Image/URL fields (array with ID+url, integer attachment ID,
5595		- * plain URL string), and recurses through repeater/group/flexible content.
5596		- *
5597		- * @param mixed $value The ACF field value (any depth)
5598		- * @param array $out Accumulator (passed by reference) for attachment IDs
5599		- * @param int $depth Recursion guard
5600		- */
5601		-private function mxchat_collect_pdf_attachment_ids_from_acf_value($value, &$out, $depth = 0) {
5602		- if ($depth > 6) {
5603		- return; // prevent runaway recursion on circular/very-deep structures
5604		- }
5605		-
5606		- if (empty($value)) {
5607		- return;
5608		- }
5609		-
5610		- // Array shapes: ACF File/Image return value=array; repeaters/groups are arrays of arrays
5611		- if (is_array($value)) {
5612		- // Direct File/Image-style array (has 'url' and usually 'ID' + 'mime_type')
5613		- $looks_like_attachment = isset($value['url']) || isset($value['ID']) || isset($value['id']);
5614		- if ($looks_like_attachment) {
5615		- $att_id = 0;
5616		- if (!empty($value['ID']) && is_numeric($value['ID'])) {
5617		- $att_id = (int) $value['ID'];
5618		- } elseif (!empty($value['id']) && is_numeric($value['id'])) {
5619		- $att_id = (int) $value['id'];
5620		- } elseif (!empty($value['url']) && is_string($value['url'])) {
5621		- $att_id = (int) attachment_url_to_postid($value['url']);
5622		- }
5623		-
5624		- $is_pdf = false;
5625		- if (!empty($value['mime_type']) && $value['mime_type'] === 'application/pdf') {
5626		- $is_pdf = true;
5627		- } elseif (!empty($value['subtype']) && strtolower((string) $value['subtype']) === 'pdf') {
5628		- $is_pdf = true;
5629		- } elseif (!empty($value['url']) && is_string($value['url']) && $this->mxchat_url_looks_like_pdf($value['url'])) {
5630		- $is_pdf = true;
5631		- } elseif ($att_id && get_post_mime_type($att_id) === 'application/pdf') {
5632		- $is_pdf = true;
5633		- }
5634		-
5635		- if ($is_pdf && $att_id && get_post_mime_type($att_id) === 'application/pdf') {
5636		- $out[] = $att_id;
5637		- }
5638		- // An array node that represents one attachment doesn't contain other
5639		- // attachments inside it — done with this branch.
5640		- return;
5641		- }
5642		-
5643		- // Recurse: repeater rows, flexible-content layouts, groups, etc.
5644		- foreach ($value as $sub) {
5645		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($sub, $out, $depth + 1);
5646		- }
5647		- return;
5648		- }
5649		-
5650		- // Plain numeric attachment ID (ACF File field set to "Return: ID")
5651		- if (is_numeric($value)) {
5652		- $att_id = (int) $value;
5653		- if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5654		- $out[] = $att_id;
5655		- }
5656		- return;
5657		- }
5658		-
5659		- // Plain string — URL pointing at a PDF (ACF File field set to "Return: URL", or a custom URL/text field)
5660		- if (is_string($value)) {
5661		- $trimmed = trim($value);
5662		- if ($trimmed !== '' && $this->mxchat_url_looks_like_pdf($trimmed)) {
5663		- $att_id = (int) attachment_url_to_postid($trimmed);
5664		- if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5665		- $out[] = $att_id;
5666		- }
5667		- }
5668		- return;
5669		- }
5670		-}
5671		-
5672		-/**
5673		- * Heuristic: does this URL/string look like a PDF reference?
5674		- * Tolerates query strings and fragments (#page=2).
5675		- */
5676		-private function mxchat_url_looks_like_pdf($url) {
5677		- if (!is_string($url) || $url === '') {
5678		- return false;
5679		- }
5680		- // Strip query + fragment before checking extension
5681		- $path = preg_replace('/[?#].*$/', '', $url);
5682		- return (bool) preg_match('/\.pdf$/i', $path);
5683		-}
5684		-
5685		-/**
5686		- * Extract text from a PDF attachment by ID using the bundled Smalot parser.
5687		- * Reads the file directly from disk via get_attached_file (no HTTP fetch).
5688		- * Result is cached on the attachment as post_meta keyed by file mtime so we
5689		- * only parse the same PDF once unless the file changes on disk.
5690		- *
5691		- * @param int $attachment_id
5692		- * @return string Extracted plain text, or '' on failure.
5693		- */
5694		-private function mxchat_extract_pdf_text_by_attachment_id($attachment_id) {
5695		- $attachment_id = (int) $attachment_id;
5696		- if ($attachment_id <= 0) {
5697		- return '';
5698		- }
5699		- if (get_post_mime_type($attachment_id) !== 'application/pdf') {
5700		- return '';
5701		- }
5702		-
5703		- $pdf_path = get_attached_file($attachment_id);
5704		- if (empty($pdf_path) || !file_exists($pdf_path) || !is_readable($pdf_path)) {
5705		- return '';
5706		- }
5707		-
5708		- // Raw-file size cap. Parsing very large PDFs can OOM the request; skip with a log entry
5709		- // and let the rest of the ACF content land in the KB. Filterable for users who need it bigger.
5710		- $default_max_bytes = 25 * 1024 * 1024;
5711		- $max_bytes = (int) apply_filters('mxchat_acf_pdf_max_bytes', $default_max_bytes, $attachment_id, $pdf_path);
5712		- if ($max_bytes > 0) {
5713		- $file_size = @filesize($pdf_path);
5714		- if ($file_size !== false && $file_size > $max_bytes) {
5715		- error_log(sprintf(
5716		- '[mxchat] ACF PDF skipped (over size cap): attachment %d "%s" %d bytes > cap %d',
5717		- $attachment_id,
5718		- basename($pdf_path),
5719		- $file_size,
5720		- $max_bytes
5721		- ));
5722		- return '';
5723		- }
5724		- }
5725		-
5726		- $mtime = @filemtime($pdf_path);
5727		- $cache_meta_key = '_mxchat_acf_pdf_text_v1';
5728		- $cached = get_post_meta($attachment_id, $cache_meta_key, true);
5729		- if (is_array($cached) && isset($cached['mtime'], $cached['text']) && (int) $cached['mtime'] === (int) $mtime) {
5730		- return (string) $cached['text'];
5731		- }
5732		-
5733		- $text = '';
5734		- try {
5735		- if (function_exists('mxchat_load_pdf_parser')) {
5736		- mxchat_load_pdf_parser();
5737		- }
5738		- if (!class_exists('\\Smalot\\PdfParser\\Parser')) {
5739		- return '';
5740		- }
5741		- $parser = new \Smalot\PdfParser\Parser();
5742		- $pdf = $parser->parseFile($pdf_path);
5743		- $pages = $pdf->getPages();
5744		- $page_texts = array();
5745		- foreach ($pages as $page) {
5746		- $page_text = '';
5747		- try {
5748		- $page_text = $page->getText();
5749		- } catch (\Exception $e) {
5750		- $page_text = '';
5751		- }
5752		- if (!empty($page_text)) {
5753		- $page_texts[] = $page_text;
5754		- }
5755		- }
5756		- $text = trim(implode("\n\n", $page_texts));
5757		- } catch (\Exception $e) {
5758		- error_log('[mxchat] ACF PDF extraction failed for attachment ' . $attachment_id . ': ' . $e->getMessage());
5759		- return '';
5760		- } catch (\Throwable $e) {
5761		- error_log('[mxchat] ACF PDF extraction error for attachment ' . $attachment_id . ': ' . $e->getMessage());
5762		- return '';
5763		- }
5764		-
5765		- // Cap per-PDF text to avoid blowing up the embedding payload on enormous PDFs.
5766		- // The chunker downstream will still split this into multiple vectors.
5767		- $max_len = (int) apply_filters('mxchat_acf_pdf_text_max_length', 50000);
5768		- if ($max_len > 0 && strlen($text) > $max_len) {
5769		- $text = substr($text, 0, $max_len);
5770		- }
5771		-
5772		- update_post_meta($attachment_id, $cache_meta_key, array(
5773		- 'mtime' => (int) $mtime,
5774		- 'text' => $text,
5775		- ));
5776		-
5777		- return $text;
5778		-}
5779		-
5780		-/**
5781		- * Handle ACF save - fires after ACF fields are saved
5782		- * This ensures ACF field data is available when syncing to knowledge base
5783		- */
5784		-public function mxchat_handle_acf_save($post_id) {
5785		- // Skip if not a valid post
5786		- if (!$post_id || $post_id === 'options') {
5787		- return;
5788		- }
5789		-
5790		- // Skip autosaves and revisions
	3024	+public function mxchat_handle_post_update($post_id, $post, $update) {
	3025	+ // Basic validation checks
5791	3026	if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5792	3027	return;
5793	3028	}
5794		-
5795		- $post = get_post($post_id);
5796		- if (!$post) {
5797		- return;
5798		- }
5799		-
5800		- $post_type = $post->post_type;
5801		-
5802		- // Check if sync is enabled for this post type
5803		- $should_sync = false;
5804		-
5805		- if ($post_type === 'post' && get_option('mxchat_auto_sync_posts') === '1') {
5806		- $should_sync = true;
5807		- } else if ($post_type === 'page' && get_option('mxchat_auto_sync_pages') === '1') {
5808		- $should_sync = true;
5809		- } else if ($post_type === 'product' && class_exists('WooCommerce')) {
5810		- // WooCommerce products - check if WooCommerce integration is enabled
5811		- $options = get_option('mxchat_options', array());
5812		- if (isset($options['enable_woocommerce_integration']) &&
5813		- ($options['enable_woocommerce_integration'] === '1' || $options['enable_woocommerce_integration'] === 'on')) {
5814		- $should_sync = true;
5815		- }
5816		- } else {
5817		- // Check custom post types
5818		- $option_name = 'mxchat_auto_sync_' . $post_type;
5819		- if (get_option($option_name) === '1') {
5820		- $should_sync = true;
5821		- }
5822		- }
5823		-
5824		- if (!$should_sync) {
5825		- return;
5826		- }
5827		-
5828		- // Only process published posts
	3029	+
	3030	+ // Only process published content
5829	3031	if ($post->post_status !== 'publish') {
5830	3032	return;
5831	3033	}
5832		-
5833		- // Check if this post has any ACF fields - if not, no need to re-sync
5834		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
5835		- if (empty($acf_fields)) {
5836		- return;
5837		- }
5838		-
5839		- // Use a transient to prevent duplicate processing (post_updated may have already run)
5840		- $transient_key = 'mxchat_acf_synced_' . $post_id;
5841		- if (get_transient($transient_key)) {
5842		- return;
5843		- }
5844		- set_transient($transient_key, true, 60); // Prevent re-processing for 60 seconds
5845		-
5846		- // Re-run the sync with ACF data now available
5847		- // We pass $update=true since this is effectively an update with ACF data
5848		- $this->mxchat_handle_post_update($post_id, $post, true);
5849		-}
5850		-
5851		-public function mxchat_handle_post_update($post_id, $post, $update) {
5852		- // Basic validation checks
5853		- if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5854		- return;
5855		- }
5856	3034
5857	3035	$post_type = $post->post_type;
5858	3036
5859	3037	// Check if sync is enabled for this post type
		@@ -5875,495 +3053,80 @@
5875	3053	if (!$should_sync) {
5876	3054	return;
5877	3055	}
5878	3056
5879		- // Check if we have stored the previous status and URL in our transients
5880		- $previous_status_key = 'mxchat_prev_status_' . $post_id;
5881		- $previous_status = get_transient($previous_status_key);
	3057	+ // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
	3058	+ $title = get_the_title($post_id);
	3059	+ $content = get_post_field('post_content', $post_id);
5882	3060
5883		- $previous_url_key = 'mxchat_prev_url_' . $post_id;
5884		- $previous_url = get_transient($previous_url_key);
	3061	+ // Apply WordPress content filters to get properly formatted content
	3062	+ $content = apply_filters('the_content', $content);
5885	3063
5886		- // If the post was previously published but is now not published, remove from knowledge base
5887		- if ($previous_status === 'publish' && $post->post_status !== 'publish') {
5888		- // Use the stored URL from when it was published, or fall back to current permalink
5889		- $source_url = $previous_url ?: get_permalink($post_id);
5890		-
5891		- // mxchat_handle_status_transition already deleted for this post earlier in this
5892		- // request (it fires first inside wp_insert_post); skip the redundant round-trip.
5893		- if ($source_url && empty($this->transition_deleted_posts[$post_id])) {
5894		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
5895		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
5896		- }
5897		-
5898		- // Clean up the transients and exit early
5899		- delete_transient($previous_status_key);
5900		- delete_transient($previous_url_key);
5901		- return;
5902		- }
5903		-
5904		- // Slug/permalink rename while still published: delete the old vectors before upserting new ones.
5905		- // Without this, md5(old_url) vectors (base + chunks) would be orphaned under the stale URL.
5906		- if ($post->post_status === 'publish' && !empty($previous_url)) {
5907		- $current_url = get_permalink($post_id);
5908		- if ($current_url && $current_url !== $previous_url) {
5909		- MxChat_Utils::delete_chunks_for_url($previous_url, 'default');
5910		- }
5911		- }
5912		-
5913		- // Store the current status for next time (if this is an update)
5914		- if ($update) {
5915		- set_transient($previous_status_key, $post->post_status, DAY_IN_SECONDS);
5916		-
5917		- // If the post is currently published, also store its URL
5918		- if ($post->post_status === 'publish') {
5919		- $current_url = get_permalink($post_id);
5920		- set_transient($previous_url_key, $current_url, DAY_IN_SECONDS);
5921		- }
5922		- }
	3064	+ // Strip tags but preserve structure
	3065	+ $content = wp_strip_all_tags($content);
5923	3066
5924		- // Only process currently published content for adding/updating.
5925		- // transition_indexed_posts: mxchat_handle_status_transition's arrival edge may have
5926		- // already indexed this post earlier in this request (editor publishes fire
5927		- // transition_post_status first, then post_updated) — skip the duplicate embed.
5928		- // Consume-once: the flag is cleared when honoured, so a LATER save of the same
5929		- // post in one long-running process (WP-CLI scripts, importers) re-indexes normally.
5930		- if ($post->post_status === 'publish') {
5931		- if (!empty($this->transition_indexed_posts[$post_id])) {
5932		- unset($this->transition_indexed_posts[$post_id]);
5933		- } else {
5934		- $this->mxchat_index_published_post($post_id, $post);
	3067	+ // Combine title and content
	3068	+ $final_content = $title . "\n\n" . $content;
	3069	+
	3070	+ // For custom post types like job_listing, include additional fields
	3071	+ if ($post_type === 'job_listing') {
	3072	+ // Add job-specific meta if available
	3073	+ $job_location = get_post_meta($post_id, '_job_location', true);
	3074	+ if (!empty($job_location)) {
	3075	+ $final_content .= "\n\nLocation: " . $job_location;
5935	3076	}
5936		- }
5937		-
5938		- // Clean up the stored previous status if not used above
5939		- if ($previous_status !== 'publish' || $post->post_status === 'publish') {
5940		- delete_transient($previous_status_key);
5941		- delete_transient($previous_url_key);
5942		- }
5943		-}
5944		-
5945		-/**
5946		- * Index a published post into the knowledge base: preprocessing filter, content
5947		- * assembly (title/excerpt/body), WooCommerce product enrichment, job_listing meta,
5948		- * ACF fields (+ optional PDF extraction), whitelisted custom meta, embedding and
5949		- * upsert, then tag-based role restriction.
5950		- *
5951		- * Shared by the post_updated auto-sync path (mxchat_handle_post_update) and the
5952		- * transition_post_status arrival edge (mxchat_handle_status_transition), so
5953		- * scheduled publishes (wp_publish_post) and direct status=publish inserts index
5954		- * identically to editor saves (plan 3055e1). Pure extraction of the former
5955		- * publish branch — body indentation retained to keep the diff reviewable.
5956		- */
5957		-private function mxchat_index_published_post($post_id, $post) {
5958		- $post_type = $post->post_type;
5959		-
5960		- // Get the source URL
5961		- $source_url = get_permalink($post_id);
5962		-
5963		- /**
5964		- * Allow developers to modify post data before processing into the knowledge base.
5965		- * Same filter and signature as the manual bulk-import path
5966		- * (ajax_mxchat_process_selected_content), so a callback registered once covers
5967		- * every indexing route. Purely additive — zero behaviour change when unhooked.
5968		- * Auto-sync runs under the 'default' bot context, matching the rest of this
5969		- * function.
5970		- *
5971		- * @param WP_Post $post The post about to be indexed.
5972		- * @param string $bot_id Bot context ('default' on auto-sync).
5973		- */
5974		- $post = apply_filters('mxchat_before_process_post', $post, 'default');
5975		- if (!($post instanceof WP_Post)) {
5976		- $post = get_post($post_id); // defend against a bad callback return
5977		- }
5978		-
5979		- // Get content with proper formatting (matching ajax_mxchat_process_selected_content),
5980		- // reading from the FILTERED post object — not re-fetched by ID, which would discard it
5981		- $title = get_the_title($post);
5982		- $content = get_post_field('post_content', $post);
5983		- $excerpt = get_post_field('post_excerpt', $post);
5984		-
5985		- // Remove shortcode tags but preserve content inside them
5986		- $content = $this->strip_shortcode_tags_preserve_content($content);
5987		- $excerpt = $this->strip_shortcode_tags_preserve_content($excerpt);
5988		-
5989		- // Strip tags but preserve structure (don't use 'the_content' filter as it may re-add shortcodes)
5990		- $content = wp_strip_all_tags($content);
5991		-
5992		- // Combine title, short description (if exists), and content
5993		- $final_content = $title . "\n\n";
5994		-
5995		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
5996		- if (!empty($excerpt)) {
5997		- $final_content .= "Short Description: " . wp_strip_all_tags($excerpt) . "\n\n";
5998		- }
5999		-
6000		- $final_content .= $content;
6001		-
6002		- // For WooCommerce products, include pricing and product details
6003		- if ($post_type === 'product' && class_exists('WooCommerce')) {
6004		- $product = wc_get_product($post_id);
6005		-
6006		- if ($product) {
6007		- // Get pricing information
6008		- $regular_price = $product->get_regular_price();
6009		- $sale_price = $product->get_sale_price();
6010		- $price = $product->get_price();
6011		- $sku = $product->get_sku();
6012		-
6013		- // Get currency symbol
6014		- $currency_symbol = get_woocommerce_currency_symbol();
6015		-
6016		- // Add pricing information
6017		- $final_content .= "\n";
6018		- if (!empty($regular_price)) {
6019		- $final_content .= "Price: " . $currency_symbol . $regular_price . "\n";
6020		- } elseif (!empty($price)) {
6021		- $final_content .= "Price: " . $currency_symbol . $price . "\n";
6022		- }
6023		-
6024		- if (!empty($sale_price) && $sale_price !== $regular_price) {
6025		- $final_content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
6026		- }
6027		-
6028		- // Handle variable products - show price range
6029		- if ($product->is_type('variable')) {
6030		- $min_price = $product->get_variation_price('min');
6031		- $max_price = $product->get_variation_price('max');
6032		- if ($min_price !== $max_price) {
6033		- $final_content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
6034		- }
6035		- }
6036		-
6037		- if (!empty($sku)) {
6038		- $final_content .= "SKU: " . $sku . "\n";
6039		- }
6040		-
6041		- // Get product categories
6042		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
6043		- if (!empty($categories) && !is_wp_error($categories)) {
6044		- $final_content .= "Categories: " . implode(', ', $categories) . "\n";
6045		- }
	3077	+
	3078	+ // Get job type terms
	3079	+ $job_types = get_the_terms($post_id, 'job_listing_type');
	3080	+ if (!empty($job_types) && !is_wp_error($job_types)) {
	3081	+ $types = array();
	3082	+ foreach ($job_types as $type) {
	3083	+ $types[] = $type->name;
6046	3084	}
	3085	+ $final_content .= "\n\nJob Type: " . implode(', ', $types);
6047	3086	}
6048		-
6049		- // For custom post types like job_listing, include additional fields
6050		- if ($post_type === 'job_listing') {
6051		- // Add job-specific meta if available
6052		- $job_location = get_post_meta($post_id, '_job_location', true);
6053		- if (!empty($job_location)) {
6054		- $final_content .= "\n\nLocation: " . $job_location;
6055		- }
6056		-
6057		- // Get job type terms
6058		- $job_types = get_the_terms($post_id, 'job_listing_type');
6059		- if (!empty($job_types) && !is_wp_error($job_types)) {
6060		- $types = array();
6061		- foreach ($job_types as $type) {
6062		- $types[] = $type->name;
6063		- }
6064		- $final_content .= "\n\nJob Type: " . implode(', ', $types);
6065		- }
6066		-
6067		- // Get company name if available
6068		- $company_name = get_post_meta($post_id, '_company_name', true);
6069		- if (!empty($company_name)) {
6070		- $final_content .= "\n\nCompany: " . $company_name;
6071		- }
6072		- }
6073		-
6074		- // ADD ACF FIELDS SUPPORT (matches ajax_mxchat_process_selected_content behavior)
6075		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
6076		- if (!empty($acf_fields)) {
6077		- $acf_content_parts = array();
6078		- $pdf_attachment_ids = array();
6079		-
6080		- foreach ($acf_fields as $field_name => $field_value) {
6081		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
6082		- if (!empty($formatted_value)) {
6083		- // Convert field name to readable label
6084		- $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name));
6085		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
6086		- }
6087		-
6088		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
6089		- }
6090		-
6091		- if (!empty($acf_content_parts)) {
6092		- $final_content .= "\n\n" . implode("\n", $acf_content_parts);
6093		- }
6094		-
6095		- // Gate the auto-sync PDF-extraction loop behind an opt-in option.
6096		- // Mirrors the per-batch checkbox the manual content selector has; the
6097		- // 25 MB size cap lives in the shared extractor so it applies in both
6098		- // paths regardless. Default OFF — re-parsing every ACF PDF on every
6099		- // editor save is expensive and most sites don't want it.
6100		- $autosync_extract_acf_pdfs = get_option('mxchat_auto_sync_acf_pdfs', '0') === '1';
6101		- if ($autosync_extract_acf_pdfs && !empty($pdf_attachment_ids)) {
6102		- $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
6103		- $pdf_sections = array();
6104		- foreach ($pdf_attachment_ids as $att_id) {
6105		- $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
6106		- if (!empty($pdf_text)) {
6107		- $pdf_title = get_the_title($att_id);
6108		- $pdf_url = wp_get_attachment_url($att_id);
6109		- $header = 'PDF Attachment';
6110		- if (!empty($pdf_title)) {
6111		- $header .= ': ' . $pdf_title;
6112		- }
6113		- if (!empty($pdf_url)) {
6114		- $header .= ' (' . $pdf_url . ')';
6115		- }
6116		- $pdf_sections[] = $header . "\n" . $pdf_text;
6117		- }
6118		- }
6119		- if (!empty($pdf_sections)) {
6120		- $final_content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
6121		- }
6122		- }
6123		- }
6124		-
6125		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
6126		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
6127		- if (!empty($custom_meta)) {
6128		- $meta_content_parts = array();
6129		-
6130		- foreach ($custom_meta as $meta_key => $meta_value) {
6131		- // Convert meta key to readable label
6132		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
6133		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
6134		- }
6135		-
6136		- if (!empty($meta_content_parts)) {
6137		- $final_content .= "\n\n" . implode("\n", $meta_content_parts);
6138		- }
6139		- }
6140		-
6141		- // Embedding decision — custom-provider-aware. Gating on a cloud API key
6142		- // here silently killed auto-sync on keyless custom-embeddings sites,
6143		- // because generate_embedding() routes custom FIRST and never needs the
6144		- // key (plan cbd5fd). Silent-return shape preserved.
6145		- $preflight = MxChat_Utils::embedding_preflight(get_option('mxchat_options'));
6146		- if (!$preflight['ok']) {
6147		- return;
6148		- }
6149		- $api_key = $preflight['api_key'];
6150		-
6151		- // Use the centralized utility function for storage
6152		- $result = MxChat_Utils::submit_content_to_db(
6153		- $final_content,
6154		- $source_url,
6155		- $api_key,
6156		- md5($source_url) // Vector ID for Pinecone
6157		- );
6158	3087
6159		- // After successful storage, apply role restriction based on tags
6160		- if (!is_wp_error($result)) {
6161		- $this->apply_role_restriction_to_post($post_id, $source_url);
	3088	+ // Get company name if available
	3089	+ $company_name = get_post_meta($post_id, '_company_name', true);
	3090	+ if (!empty($company_name)) {
	3091	+ $final_content .= "\n\nCompany: " . $company_name;
6162	3092	}
6163		-}
6164		-
6165		-/**
6166		- * Store the post status and URL before update to detect status transitions
6167		- * This runs before the post is actually updated in the database
6168		- */
6169		-public function mxchat_store_pre_update_status($post_id, $data) {
6170		- // Get the current post from database (before update)
6171		- $current_post = get_post($post_id);
	3093	+ }
6172	3094
6173		- if ($current_post) {
6174		- // Store the current status temporarily
6175		- $status_key = 'mxchat_prev_status_' . $post_id;
6176		- set_transient($status_key, $current_post->post_status, HOUR_IN_SECONDS);
6177		-
6178		- // If the post is currently published, also store its URL
6179		- if ($current_post->post_status === 'publish') {
6180		- $url_key = 'mxchat_prev_url_' . $post_id;
6181		- $current_url = get_permalink($post_id);
6182		- set_transient($url_key, $current_url, HOUR_IN_SECONDS);
6183		- }
	3095	+ // Get the source URL
	3096	+ $source_url = get_permalink($post_id);
	3097	+
	3098	+ // Get API key with proper model detection
	3099	+ $options = get_option('mxchat_options');
	3100	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	3101	+
	3102	+ if (strpos($selected_model, 'voyage') === 0) {
	3103	+ $api_key = $options['voyage_api_key'] ?? '';
	3104	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	3105	+ $api_key = $options['gemini_api_key'] ?? '';
	3106	+ } else {
	3107	+ $api_key = $options['api_key'] ?? '';
6184	3108	}
6185		-}
6186		-
6187		-/**
6188		- * Whether auto-sync is enabled for a post type (mirrors the checks used by the
6189		- * update/delete handlers; kept as one helper so new call sites cannot drift).
6190		- */
6191		-private function mxchat_is_auto_sync_enabled($post_type) {
6192		- if ($post_type === 'post') {
6193		- return get_option('mxchat_auto_sync_posts') === '1';
6194		- }
6195		- if ($post_type === 'page') {
6196		- return get_option('mxchat_auto_sync_pages') === '1';
6197		- }
6198		- return get_option('mxchat_auto_sync_' . $post_type) === '1';
6199		-}
6200		-
6201		-/**
6202		- * Remove a post's vectors the moment it leaves 'publish', using the authoritative
6203		- * old status core passes to transition_post_status — no transient involved (plan 816fb1).
6204		- *
6205		- * Covers status changes that never route through wp_update_post (scheduled-expiry
6206		- * plugins and others that flip post_status directly and call wp_transition_post_status),
6207		- * where neither pre_post_update nor post_updated fires and the old detection missed.
6208		- */
6209		-public function mxchat_handle_status_transition($new_status, $old_status, $post) {
6210		- if (!($post instanceof WP_Post) || wp_is_post_revision($post->ID)) {
	3109	+
	3110	+ if (empty($api_key)) {
	3111	+ //error_log('MxChat Auto-sync: No API key configured for embedding model');
6211	3112	return;
6212	3113	}
6213		-
6214		- // Arrival edge (plan 3055e1): a post BECOMING published is indexed here, because
6215		- // wp_publish_post() — the path scheduled posts take via check_and_publish_future_post —
6216		- // and direct wp_insert_post(status=publish) creates never fire post_updated, so the
6217		- // auto-sync ADD path alone misses them. Editor publishes also pass through here;
6218		- // the transition_indexed_posts guard keeps mxchat_handle_post_update from embedding
6219		- // a second time in the same request.
6220		- if ($new_status === 'publish' && $old_status !== 'publish') {
6221		- if ($this->mxchat_is_auto_sync_enabled($post->post_type)) {
6222		- $this->mxchat_index_published_post($post->ID, $post);
6223		- $this->transition_indexed_posts[$post->ID] = true;
6224		- }
6225		- return;
	3114	+
	3115	+ // Use the centralized utility function for storage
	3116	+ $result = MxChat_Utils::submit_content_to_db(
	3117	+ $final_content,
	3118	+ $source_url,
	3119	+ $api_key,
	3120	+ md5($source_url) // Vector ID for Pinecone
	3121	+ );
	3122	+
	3123	+ if (is_wp_error($result)) {
	3124	+ //error_log('MxChat Auto-sync failed for post ' . $post_id . ': ' . $result->get_error_message());
6226	3125	}
6227		-
6228		- // Only the publish -> not-publish edge matters here.
6229		- if ($old_status !== 'publish' || $new_status === 'publish') {
6230		- return;
6231		- }
6232		- // Trash is handled by mxchat_handle_post_delete (wp_trash_post) with pre-trash URL
6233		- // resolution; skip to avoid a second network round-trip per trash.
6234		- if ($new_status === 'trash') {
6235		- return;
6236		- }
6237		- if (!$this->mxchat_is_auto_sync_enabled($post->post_type)) {
6238		- return;
6239		- }
6240		-
6241		- $urls = array();
6242		-
6243		- // The DB may already hold the new status when this fires, so get_permalink() on the
6244		- // live post could build a draft-style URL whose md5 misses the stored vector IDs.
6245		- // Reconstruct the published permalink from a clone instead.
6246		- $published_clone = clone $post;
6247		- $published_clone->post_status = 'publish';
6248		- $published_url = get_permalink($published_clone);
6249		- if ($published_url) {
6250		- $urls[] = $published_url;
6251		- }
6252		-
6253		- // Honour the pre-update capture when present (covers a slug change in the same save).
6254		- $previous_url = get_transient('mxchat_prev_url_' . $post->ID);
6255		- if (!empty($previous_url)) {
6256		- $urls[] = $previous_url;
6257		- }
6258		-
6259		- foreach (array_unique($urls) as $url) {
6260		- MxChat_Utils::delete_chunks_for_url($url, 'default');
6261		- }
6262		-
6263		- if (!empty($urls)) {
6264		- $this->transition_deleted_posts[$post->ID] = true;
6265		- }
6266	3126	}
6267	3127
6268		-/**
6269		- * WP-CLI: remove knowledge-base entries left behind by posts that were unpublished,
6270		- * trashed, or made private before the transition_post_status handler existed.
6271		- *
6272		- * Walks every auto-synced post type's non-published posts, reconstructs each one's
6273		- * published-era permalink, and deletes its vectors (routes to Pinecone or the WP table).
6274		- * Deletion is idempotent, so never-indexed posts are a cheap no-op.
6275		- *
6276		- * ## OPTIONS
6277		- *
6278		- * [--dry-run]
6279		- * : Report what would be removed without deleting anything.
6280		- *
6281		- * ## EXAMPLES
6282		- *
6283		- * wp mxchat prune-unpublished --dry-run
6284		- * wp mxchat prune-unpublished
6285		- */
6286		-public function cli_prune_unpublished($args, $assoc_args) {
6287		- global $wpdb;
6288		- $dry_run = !empty($assoc_args['dry-run']);
6289		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
6290	3128
6291		- $candidate_types = array_merge(array('post', 'page'), array_values(get_post_types(array('_builtin' => false), 'names')));
6292		- $synced_types = array();
6293		- foreach ($candidate_types as $type) {
6294		- if ($this->mxchat_is_auto_sync_enabled($type)) {
6295		- $synced_types[] = $type;
6296		- }
6297		- }
6298		- if (empty($synced_types)) {
6299		- WP_CLI::success('No post types have auto-sync enabled; nothing to prune.');
6300		- return;
6301		- }
6302		-
6303		- $scanned = 0;
6304		- $pruned = 0;
6305		- $paged = 1;
6306		- do {
6307		- $query = new WP_Query(array(
6308		- 'post_type' => $synced_types,
6309		- 'post_status' => array('draft', 'pending', 'private', 'future', 'trash'),
6310		- 'posts_per_page' => 100,
6311		- 'paged' => $paged,
6312		- 'fields' => 'ids',
6313		- ));
6314		- foreach ($query->posts as $post_id) {
6315		- $post = get_post($post_id);
6316		- if (!$post) {
6317		- continue;
6318		- }
6319		- $scanned++;
6320		-
6321		- // Rebuild the permalink the post had while published: publish-status clone,
6322		- // with wp_trash_post's __trashed slug suffix stripped for trashed posts.
6323		- $clone = clone $post;
6324		- $clone->post_status = 'publish';
6325		- if (substr($clone->post_name, -9) === '__trashed') {
6326		- $clone->post_name = substr($clone->post_name, 0, -9);
6327		- }
6328		- $url = get_permalink($clone);
6329		- if (!$url) {
6330		- continue;
6331		- }
6332		-
6333		- // Local-table row count is exact in WordPress-DB mode; in Pinecone mode it
6334		- // reads 0 but the delete below still routes to Pinecone and is idempotent.
6335		- $local_rows = (int) $wpdb->get_var($wpdb->prepare(
6336		- "SELECT COUNT(*) FROM {$table} WHERE source_url = %s", $url
6337		- ));
6338		-
6339		- if ($dry_run) {
6340		- if ($local_rows > 0) {
6341		- WP_CLI::log(sprintf('Would remove %d row(s): %s (post %d, %s)', $local_rows, $url, $post_id, $post->post_status));
6342		- $pruned += $local_rows;
6343		- }
6344		- continue;
6345		- }
6346		-
6347		- MxChat_Utils::delete_chunks_for_url($url, 'default');
6348		- if ($local_rows > 0) {
6349		- WP_CLI::log(sprintf('Removed %d row(s): %s (post %d, %s)', $local_rows, $url, $post_id, $post->post_status));
6350		- $pruned += $local_rows;
6351		- }
6352		- }
6353		- $more = $paged < $query->max_num_pages;
6354		- $paged++;
6355		- } while ($more);
6356		-
6357		- WP_CLI::success(sprintf(
6358		- '%s %d local knowledge row(s) across %d non-published post(s) scanned.%s',
6359		- $dry_run ? 'Would remove' : 'Removed',
6360		- $pruned,
6361		- $scanned,
6362		- ' (Pinecone-mode deletions are not counted locally.)'
6363		- ));
6364		-}
6365		-
6366	3129	public function mxchat_handle_post_delete($post_id) {
6367	3130	// Get post data before it's deleted
6368	3131	$post = get_post($post_id);
6369	3132
		@@ -6393,51 +3156,85 @@
6393	3156	if (!$should_sync) {
6394	3157	return;
6395	3158	}
6396	3159
6397		- // Resolve the pre-trash URL. wp_trash_post renames the slug with "__trashed" before firing
6398		- // this hook, so get_permalink() here would return the trashed URL and md5() would miss the
6399		- // real vector IDs stored under the original URL.
6400		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
	3160	+ // Get the URL before post is deleted
	3161	+ $source_url = get_permalink($post_id);
6401	3162	if (!$source_url) {
6402		- //error_log('MXChat: Failed to resolve source URL for post ' . $post_id);
	3163	+ //error_log('MXChat: Failed to get permalink for post ' . $post_id);
6403	3164	return;
6404	3165	}
6405	3166
6406		- // Use chunk-aware deletion (handles both chunked and non-chunked content)
6407		- $delete_result = MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	3167	+ // Check if Pinecone is enabled
	3168	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	3169	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6408	3170
6409		- if (is_wp_error($delete_result)) {
6410		- //error_log('MXChat: Chunk-aware deletion failed for URL: ' . $source_url . ' - ' . $delete_result->get_error_message());
	3171	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3172	+ // Delete from Pinecone
	3173	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
	3174	+ } else {
	3175	+ // Delete from WordPress DB
	3176	+ global $wpdb;
	3177	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	3178	+
	3179	+ $result = $wpdb->delete(
	3180	+ $table_name,
	3181	+ array('source_url' => $source_url),
	3182	+ array('%s')
	3183	+ );
	3184	+
	3185	+ if ($result === false) {
	3186	+ //error_log('MXChat: WordPress DB deletion failed for URL: ' . $source_url);
	3187	+ }
6411	3188	}
	3189	+}
6412	3190
6413		- delete_transient('mxchat_prev_url_' . $post_id);
6414		- delete_transient('mxchat_prev_status_' . $post_id);
6415		-}
6416	3191
6417		-/**
6418		- * Resolve the source URL for a post being trashed/deleted.
6419		- *
6420		- * Why: wp_trash_post appends "__trashed" to the slug before the wp_trash_post action fires, so
6421		- * get_permalink() returns a URL whose md5() won't match the vector IDs stored in Pinecone or
6422		- * the source_url rows in the WP DB. Prefer the URL captured by mxchat_store_pre_update_status
6423		- * (runs on pre_post_update, before the rename); fall back to stripping the __trashed suffix.
6424		- */
6425		-private function mxchat_resolve_pre_trash_url($post_id) {
6426		- $previous_url = get_transient('mxchat_prev_url_' . $post_id);
6427		- if (!empty($previous_url)) {
6428		- return $previous_url;
6429		- }
	3192	+ /**
	3193	+ * Deletes data from Pinecone using a source URL
	3194	+ */
	3195	+ public function mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options) {
	3196	+ $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
	3197	+ $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
6430	3198
6431		- $current = get_permalink($post_id);
6432		- if (!$current) {
6433		- return '';
6434		- }
6435		- return preg_replace('#__trashed(/?)$#', '$1', $current);
6436		-}
	3199	+ if (empty($host) || empty($api_key)) {
	3200	+ //error_log('MXChat: Pinecone deletion failed - missing configuration');
	3201	+ return false;
	3202	+ }
6437	3203
	3204	+ $api_endpoint = "https://{$host}/vectors/delete";
	3205	+ $vector_id = md5($source_url);
6438	3206
	3207	+ $request_body = array(
	3208	+ 'ids' => array($vector_id)
	3209	+ );
6439	3210
	3211	+ $response = wp_remote_post($api_endpoint, array(
	3212	+ 'headers' => array(
	3213	+ 'Api-Key' => $api_key,
	3214	+ 'accept' => 'application/json',
	3215	+ 'content-type' => 'application/json'
	3216	+ ),
	3217	+ 'body' => wp_json_encode($request_body),
	3218	+ 'timeout' => 30
	3219	+ ));
	3220	+
	3221	+ if (is_wp_error($response)) {
	3222	+ //error_log('MXChat: Pinecone deletion error - ' . $response->get_error_message());
	3223	+ return false;
	3224	+ }
	3225	+
	3226	+ $response_code = wp_remote_retrieve_response_code($response);
	3227	+ if ($response_code !== 200) {
	3228	+ //error_log('MXChat: Pinecone deletion failed with status ' . $response_code);
	3229	+ return false;
	3230	+ }
	3231	+
	3232	+ return true;
	3233	+ }
	3234	+
	3235	+
	3236	+
6440	3237	public function mxchat_handle_product_change($post_id, $post, $update) {
6441	3238	if ($post->post_type !== 'product') {
6442	3239	return;
6443	3240	}
		@@ -6459,12 +3256,11 @@
6459	3256	if (!isset($this->options['enable_woocommerce_integration']) ||
6460	3257	!in_array($this->options['enable_woocommerce_integration'], ['1', 'on'])) {
6461	3258	return;
6462	3259	}
6463		-
	3260	+
6464	3261	$source_url = get_permalink($product->get_id());
6465		- $product_id = $product->get_id();
6466		-
	3262	+
6467	3263	// Build product content
6468	3264	$title = $product->get_name();
6469	3265	$description = $product->get_description();
6470	3266	$short_description = $product->get_short_description();
		@@ -6469,97 +3265,48 @@
6469	3265	$description = $product->get_description();
6470	3266	$short_description = $product->get_short_description();
6471	3267	$regular_price = $product->get_regular_price();
6472	3268	$sale_price = $product->get_sale_price();
6473		- $price = $product->get_price();
6474	3269	$sku = $product->get_sku();
6475		-
6476		- // Get currency symbol
6477		- $currency_symbol = get_woocommerce_currency_symbol();
6478		-
	3270	+
6479	3271	// Format content consistently
6480	3272	$content = $title . "\n\n";
6481		-
6482		- if (!empty($short_description)) {
6483		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
6484		- }
6485		-
	3273	+
6486	3274	if (!empty($description)) {
6487	3275	$content .= wp_strip_all_tags($description) . "\n\n";
6488	3276	}
6489		-
6490		- // Add pricing information
6491		- if (!empty($regular_price)) {
6492		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
6493		- } elseif (!empty($price)) {
6494		- $content .= "Price: " . $currency_symbol . $price . "\n";
	3277	+
	3278	+ if (!empty($short_description)) {
	3279	+ $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
6495	3280	}
6496		-
6497		- if (!empty($sale_price) && $sale_price !== $regular_price) {
6498		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
	3281	+
	3282	+ $content .= "Price: $" . $regular_price . "\n";
	3283	+
	3284	+ if (!empty($sale_price)) {
	3285	+ $content .= "Sale Price: $" . $sale_price . "\n";
6499	3286	}
6500		-
6501		- // Handle variable products - show price range
6502		- if ($product->is_type('variable')) {
6503		- $min_price = $product->get_variation_price('min');
6504		- $max_price = $product->get_variation_price('max');
6505		- if ($min_price !== $max_price) {
6506		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
6507		- }
6508		- }
6509		-
	3287	+
6510	3288	if (!empty($sku)) {
6511	3289	$content .= "SKU: " . $sku . "\n";
6512	3290	}
6513		-
6514		- // Get product categories
6515		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
6516		- if (!empty($categories) && !is_wp_error($categories)) {
6517		- $content .= "Categories: " . implode(', ', $categories) . "\n";
	3291	+
	3292	+ // Get API key with proper model detection
	3293	+ $options = get_option('mxchat_options');
	3294	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	3295	+
	3296	+ if (strpos($selected_model, 'voyage') === 0) {
	3297	+ $api_key = $options['voyage_api_key'] ?? '';
	3298	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	3299	+ $api_key = $options['gemini_api_key'] ?? '';
	3300	+ } else {
	3301	+ $api_key = $options['api_key'] ?? '';
6518	3302	}
6519		-
6520		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
6521		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
6522		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
6523		- foreach ($custom_tabs as $tab) {
6524		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
6525		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
6526		-
6527		- if (!empty($tab_title) && !empty($tab_content)) {
6528		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
6529		- }
6530		- }
6531		- }
6532		-
6533		- // Also check for reusable/saved tabs applied to this product
6534		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
6535		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
6536		- // Get the saved tabs option
6537		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
6538		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
6539		- foreach ($applied_saved_tabs as $saved_tab_id) {
6540		- if (isset($saved_tabs[$saved_tab_id])) {
6541		- $tab = $saved_tabs[$saved_tab_id];
6542		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
6543		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
6544		-
6545		- if (!empty($tab_title) && !empty($tab_content)) {
6546		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
6547		- }
6548		- }
6549		- }
6550		- }
6551		- }
6552		-
6553		- // Embedding decision — custom-provider-aware (plan cbd5fd); silent-return
6554		- // shape preserved.
6555		- $preflight = MxChat_Utils::embedding_preflight(get_option('mxchat_options'));
6556		- if (!$preflight['ok']) {
6557		- //error_log('MxChat Auto-sync: embedding pre-flight failed: ' . $preflight['reason']);
	3303	+
	3304	+ if (empty($api_key)) {
	3305	+ //error_log('MxChat Auto-sync: No API key configured for embedding model');
6558	3306	return;
6559	3307	}
6560		- $api_key = $preflight['api_key'];
6561		-
	3308	+
6562	3309	// Use the centralized utility function for storage
6563	3310	$result = MxChat_Utils::submit_content_to_db(
6564	3311	$content,
6565	3312	$source_url,
		@@ -6565,16 +3312,11 @@
6565	3312	$source_url,
6566	3313	$api_key,
6567	3314	md5($source_url) // Vector ID for Pinecone
6568	3315	);
6569		-
6570		- // After successful storage, apply role restriction based on tags
6571		- if (!is_wp_error($result)) {
6572		- $this->apply_role_restriction_to_post($product_id, $source_url);
6573		- }
6574		-
	3316	+
6575	3317	if (is_wp_error($result)) {
6576		- //error_log('MxChat WooCommerce sync failed for product ' . $product_id . ': ' . $result->get_error_message());
	3318	+ //error_log('MxChat WooCommerce sync failed for product ' . $product->get_id() . ': ' . $result->get_error_message());
6577	3319	}
6578	3320	}
6579	3321
6580	3322	public function mxchat_handle_product_delete($post_id) {
		@@ -6581,18 +3323,28 @@
6581	3323	if (get_post_type($post_id) !== 'product') {
6582	3324	return;
6583	3325	}
6584	3326
6585		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
6586		- if (!$source_url) {
6587		- return;
6588		- }
	3327	+ $source_url = get_permalink($post_id);
6589	3328
6590		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
6591		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	3329	+ // Check if Pinecone is enabled
	3330	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	3331	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6592	3332
6593		- delete_transient('mxchat_prev_url_' . $post_id);
6594		- delete_transient('mxchat_prev_status_' . $post_id);
	3333	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3334	+ // Delete from Pinecone
	3335	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
	3336	+ } else {
	3337	+ // Delete from WordPress DB
	3338	+ global $wpdb;
	3339	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	3340	+
	3341	+ $wpdb->delete(
	3342	+ $table_name,
	3343	+ array('source_url' => $source_url),
	3344	+ array('%s')
	3345	+ );
	3346	+ }
6595	3347	}
6596	3348
6597	3349	/**
6598	3350	* Handle individual Pinecone content deletion
		@@ -6640,9 +3392,21 @@
6640	3392	$pinecone_options['mxchat_pinecone_host']
6641	3393	);
6642	3394
6643	3395	if ($result['success']) {
6644		- // No cache clearing needed since we removed caching
	3396	+ // Remove from ALL caches
	3397	+ $pinecone_manager->mxchat_remove_from_pinecone_vector_cache($vector_id);
	3398	+ $pinecone_manager->mxchat_remove_from_processed_content_caches($vector_id);
	3399	+
	3400	+ // CLEAR ALL RELEVANT CACHES
	3401	+ delete_transient('mxchat_pinecone_recent_1k_cache');
	3402	+ delete_option('mxchat_pinecone_vector_ids_cache');
	3403	+ delete_option('mxchat_pinecone_processed_cache');
	3404	+ delete_option('mxchat_processed_content_cache');
	3405	+
	3406	+ // Also force refresh for next page load
	3407	+ $pinecone_manager->mxchat_refresh_after_new_content($pinecone_options);
	3408	+
6645	3409	set_transient('mxchat_admin_notice_success',
6646	3410	esc_html__('Entry deleted successfully from Pinecone.', 'mxchat'),
6647	3411	30
6648	3412	);
		@@ -6655,11 +3419,9 @@
6655	3419
6656	3420	wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
6657	3421	exit;
6658	3422	}
6659		-/**
6660		- * Handle individual Pinecone content deletion via AJAX
6661		- */
	3423	+
6662	3424	public function ajax_mxchat_delete_pinecone_prompt() {
6663	3425	// Verify nonce and permissions
6664	3426	if (!check_ajax_referer('mxchat_delete_pinecone_prompt_nonce', 'nonce', false)) {
6665	3427	wp_send_json_error('Invalid nonce');
		@@ -6671,9 +3433,8 @@
6671	3433	exit;
6672	3434	}
6673	3435
6674	3436	$vector_id = isset($_POST['vector_id']) ? sanitize_text_field($_POST['vector_id']) : '';
6675		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6676	3437
6677	3438	if (empty($vector_id)) {
6678	3439	wp_send_json_error('Missing vector ID');
6679	3440	exit;
		@@ -6678,20 +3439,19 @@
6678	3439	wp_send_json_error('Missing vector ID');
6679	3440	exit;
6680	3441	}
6681	3442
6682		- // Get bot-specific Pinecone settings
6683		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6684		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6685		-
	3443	+ // Get Pinecone settings
	3444	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6686	3445	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6687	3446
6688	3447	if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
6689		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
	3448	+ wp_send_json_error('Pinecone is not properly configured');
6690	3449	exit;
6691	3450	}
6692	3451
6693		- // Delete from the correct Pinecone index
	3452	+ // Delete from Pinecone
	3453	+ $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6694	3454	$result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
6695	3455	$vector_id,
6696	3456	$pinecone_options['mxchat_pinecone_api_key'],
6697	3457	$pinecone_options['mxchat_pinecone_host']
		@@ -6697,1007 +3457,32 @@
6697	3457	$pinecone_options['mxchat_pinecone_host']
6698	3458	);
6699	3459
6700	3460	if ($result['success']) {
6701		- // No cache clearing needed since we removed caching
	3461	+ // Remove from ALL caches
	3462	+ $pinecone_manager->mxchat_remove_from_pinecone_vector_cache($vector_id);
	3463	+ $pinecone_manager->mxchat_remove_from_processed_content_caches($vector_id);
	3464	+
	3465	+ // CLEAR ALL RELEVANT CACHES (ADD THESE LINES)
	3466	+ delete_transient('mxchat_pinecone_recent_1k_cache');
	3467	+ delete_option('mxchat_pinecone_vector_ids_cache');
	3468	+ delete_option('mxchat_pinecone_processed_cache');
	3469	+ delete_option('mxchat_processed_content_cache');
	3470	+
	3471	+ // Also force refresh for next page load
	3472	+ $pinecone_manager->mxchat_refresh_after_new_content($pinecone_options);
	3473	+
6702	3474	wp_send_json_success(array(
6703	3475	'message' => 'Entry deleted successfully from Pinecone',
6704		- 'vector_id' => $vector_id,
6705		- 'bot_id' => $bot_id
	3476	+ 'vector_id' => $vector_id
6706	3477	));
6707	3478	} else {
6708		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete from Pinecone: ' . $result['message'], array('vector_id' => $vector_id, 'bot_id' => $bot_id));
6709	3479	wp_send_json_error('Failed to delete from Pinecone: ' . $result['message']);
6710	3480	}
6711	3481
6712	3482	exit;
6713	3483	}
6714		-
6715		-/**
6716		- * Handle deletion of all chunks for a given source URL via AJAX
6717		- * Follows the same pattern as ajax_mxchat_delete_pinecone_prompt
6718		- */
6719		-public function ajax_mxchat_delete_chunks_by_url() {
6720		- // Verify nonce and permissions
6721		- if (!check_ajax_referer('mxchat_delete_chunks_nonce', 'nonce', false)) {
6722		- wp_send_json_error('Invalid nonce');
6723		- exit;
6724		- }
6725		-
6726		- if (!current_user_can('manage_options')) {
6727		- wp_send_json_error('Unauthorized access');
6728		- exit;
6729		- }
6730		-
6731		- $source_url = isset($_POST['source_url']) ? esc_url_raw($_POST['source_url']) : '';
6732		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
6733		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6734		-
6735		- if (empty($source_url)) {
6736		- wp_send_json_error('Missing source URL');
6737		- exit;
6738		- }
6739		-
6740		- // Generate the base vector ID from the source URL (same as how chunks are created)
6741		- $base_vector_id = md5($source_url);
6742		-
6743		- if ($data_source === 'pinecone') {
6744		- // Get bot-specific Pinecone settings (same as working delete function)
6745		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6746		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6747		-
6748		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6749		-
6750		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
6751		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
6752		- exit;
6753		- }
6754		-
6755		- $api_key = $pinecone_options['mxchat_pinecone_api_key'];
6756		- $host = $pinecone_options['mxchat_pinecone_host'];
6757		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
6758		-
6759		- // Collect all vector IDs to delete
6760		- $vectors_to_delete = array();
6761		-
6762		- // Add the original single-vector ID (for non-chunked content)
6763		- $vectors_to_delete[] = $base_vector_id;
6764		-
6765		- // Use Pinecone list API to find all chunk vectors with this prefix
6766		- // NOTE: Pinecone List API is a GET request with query parameters, not POST
6767		- $prefix = $base_vector_id . '_chunk_';
6768		-
6769		- $query_params = array(
6770		- 'prefix' => $prefix,
6771		- 'limit' => 100
6772		- );
6773		-
6774		- if (!empty($namespace)) {
6775		- $query_params['namespace'] = $namespace;
6776		- }
6777		-
6778		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
6779		-
6780		- $list_response = wp_remote_get($list_url, array(
6781		- 'headers' => array(
6782		- 'Api-Key' => $api_key,
6783		- 'accept' => 'application/json'
6784		- ),
6785		- 'timeout' => 30
6786		- ));
6787		-
6788		- if (!is_wp_error($list_response)) {
6789		- $list_body_response = wp_remote_retrieve_body($list_response);
6790		- $list_data = json_decode($list_body_response, true);
6791		- if (!empty($list_data['vectors'])) {
6792		- foreach ($list_data['vectors'] as $vector) {
6793		- if (isset($vector['id'])) {
6794		- $vectors_to_delete[] = $vector['id'];
6795		- }
6796		- }
6797		- }
6798		- }
6799		-
6800		- if (empty($vectors_to_delete)) {
6801		- wp_send_json_success(array(
6802		- 'message' => 'No vectors found to delete',
6803		- 'source_url' => $source_url
6804		- ));
6805		- exit;
6806		- }
6807		-
6808		- // Delete all vectors using the same endpoint as the working function
6809		- $delete_url = "https://{$host}/vectors/delete";
6810		-
6811		- $delete_body = array(
6812		- 'ids' => $vectors_to_delete
6813		- );
6814		-
6815		- if (!empty($namespace)) {
6816		- $delete_body['namespace'] = $namespace;
6817		- }
6818		-
6819		- $delete_response = wp_remote_post($delete_url, array(
6820		- 'headers' => array(
6821		- 'Api-Key' => $api_key,
6822		- 'accept' => 'application/json',
6823		- 'content-type' => 'application/json'
6824		- ),
6825		- 'body' => wp_json_encode($delete_body),
6826		- 'timeout' => 30
6827		- ));
6828		-
6829		- if (is_wp_error($delete_response)) {
6830		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete chunks from Pinecone: ' . $delete_response->get_error_message(), array('source_url' => $source_url));
6831		- wp_send_json_error('Failed to delete from Pinecone: ' . $delete_response->get_error_message());
6832		- exit;
6833		- }
6834		-
6835		- $response_code = wp_remote_retrieve_response_code($delete_response);
6836		-
6837		- if ($response_code !== 200) {
6838		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Pinecone API error (HTTP ' . $response_code . ')', array('source_url' => $source_url));
6839		- wp_send_json_error('Pinecone API error (HTTP ' . $response_code . ')');
6840		- exit;
6841		- }
6842		-
6843		- wp_send_json_success(array(
6844		- 'message' => 'All chunks deleted successfully from Pinecone',
6845		- 'source_url' => $source_url,
6846		- 'deleted_count' => count($vectors_to_delete)
6847		- ));
6848		-
6849		- } else {
6850		- // WordPress database deletion
6851		- global $wpdb;
6852		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6853		-
6854		- $result = $wpdb->delete(
6855		- $table_name,
6856		- array('source_url' => $source_url),
6857		- array('%s')
6858		- );
6859		-
6860		- if ($result === false) {
6861		- MxChat_Admin::mxchat_log_debug('database_error', 'Failed to delete from database: ' . $wpdb->last_error, array('source_url' => $source_url));
6862		- wp_send_json_error('Failed to delete from database: ' . $wpdb->last_error);
6863		- exit;
6864		- }
6865		-
6866		- wp_send_json_success(array(
6867		- 'message' => 'All chunks deleted successfully from database',
6868		- 'source_url' => $source_url,
6869		- 'deleted_count' => $result
6870		- ));
6871		- }
6872		-
6873		- exit;
6874		-}
6875		-
6876		-/**
6877		- * Handle individual WordPress database content deletion via AJAX
6878		- * Mirrors the Pinecone delete handler but for WordPress database entries
6879		- */
6880		-public function ajax_mxchat_delete_wordpress_prompt() {
6881		- // Verify nonce and permissions
6882		- if (!check_ajax_referer('mxchat_delete_wordpress_prompt_nonce', 'nonce', false)) {
6883		- wp_send_json_error('Invalid nonce');
6884		- exit;
6885		- }
6886		-
6887		- if (!current_user_can('manage_options')) {
6888		- wp_send_json_error('Unauthorized access');
6889		- exit;
6890		- }
6891		-
6892		- $entry_id = isset($_POST['entry_id']) ? intval($_POST['entry_id']) : 0;
6893		-
6894		- if (empty($entry_id)) {
6895		- wp_send_json_error('Missing entry ID');
6896		- exit;
6897		- }
6898		-
6899		- global $wpdb;
6900		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6901		-
6902		- // Clear cache for this entry
6903		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
6904		-
6905		- // Delete from database
6906		- $result = $wpdb->delete(
6907		- $table_name,
6908		- array('id' => $entry_id),
6909		- array('%d')
6910		- );
6911		-
6912		- if ($result !== false) {
6913		- wp_send_json_success(array(
6914		- 'message' => 'Entry deleted successfully',
6915		- 'entry_id' => $entry_id
6916		- ));
6917		- } else {
6918		- wp_send_json_error('Failed to delete entry from database');
6919		- }
6920		-
6921		- exit;
6922		-}
6923		-
6924		-/**
6925		- * Handle bulk deletion of knowledge entries via AJAX
6926		- * Supports both Pinecone and WordPress database entries
6927		- */
6928		-public function ajax_mxchat_bulk_delete_knowledge() {
6929		- // Verify nonce and permissions
6930		- if (!check_ajax_referer('mxchat_bulk_delete_knowledge_nonce', 'nonce', false)) {
6931		- wp_send_json_error('Invalid nonce');
6932		- exit;
6933		- }
6934		-
6935		- if (!current_user_can('manage_options')) {
6936		- wp_send_json_error('Unauthorized access');
6937		- exit;
6938		- }
6939		-
6940		- $entries = isset($_POST['entries']) ? $_POST['entries'] : array();
6941		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6942		-
6943		- if (empty($entries) || !is_array($entries)) {
6944		- wp_send_json_error('No entries provided');
6945		- exit;
6946		- }
6947		-
6948		- // Extend execution time — bulk Pinecone operations can take a while
6949		- if (function_exists('set_time_limit')) {
6950		- set_time_limit(120);
6951		- }
6952		-
6953		- $success_ids = array();
6954		- $failed_ids = array();
6955		- $errors = array();
6956		-
6957		- global $wpdb;
6958		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6959		-
6960		- // Get Pinecone manager for Pinecone deletions
6961		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6962		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6963		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6964		-
6965		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
6966		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
6967		-
6968		- // =============================================
6969		- // PHASE 1: Collect all Pinecone vector IDs
6970		- // and separate WordPress entries
6971		- // =============================================
6972		- $pinecone_entry_ids = array(); // entry IDs that are pinecone-sourced
6973		- $wordpress_entries = array(); // entries for WordPress DB deletion
6974		- $all_vector_ids = array(); // all pinecone vector IDs to delete in one batch
6975		-
6976		- foreach ($entries as $entry) {
6977		- $entry_id = sanitize_text_field($entry['id'] ?? '');
6978		- $source = sanitize_text_field($entry['source'] ?? 'wordpress');
6979		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
6980		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
6981		-
6982		- if (empty($entry_id)) {
6983		- continue;
6984		- }
6985		-
6986		- if ($source === 'pinecone') {
6987		- if (!$use_pinecone || empty($api_key)) {
6988		- $failed_ids[] = $entry_id;
6989		- $errors[] = "Pinecone not configured for entry: $entry_id";
6990		- continue;
6991		- }
6992		-
6993		- $pinecone_entry_ids[] = $entry_id;
6994		-
6995		- if ($is_group && !empty($source_url)) {
6996		- // Grouped/chunked entry: collect base ID + chunk IDs via List API
6997		- $base_vector_id = md5($source_url);
6998		- $all_vector_ids[] = $base_vector_id;
6999		-
7000		- $list_url = 'https://' . $host . '/vectors/list?prefix=' . urlencode($base_vector_id . '_chunk_') . '&limit=100';
7001		- $list_response = wp_remote_get($list_url, array(
7002		- 'headers' => array(
7003		- 'Api-Key' => $api_key,
7004		- 'accept' => 'application/json'
7005		- ),
7006		- 'timeout' => 30
7007		- ));
7008		-
7009		- if (!is_wp_error($list_response)) {
7010		- $list_body = json_decode(wp_remote_retrieve_body($list_response), true);
7011		- if (!empty($list_body['vectors']) && is_array($list_body['vectors'])) {
7012		- foreach ($list_body['vectors'] as $vector) {
7013		- if (isset($vector['id'])) {
7014		- $all_vector_ids[] = $vector['id'];
7015		- }
7016		- }
7017		- }
7018		- }
7019		- } else {
7020		- // Single entry: the entry_id IS the vector ID
7021		- $all_vector_ids[] = $entry_id;
7022		- }
7023		- } else {
7024		- $wordpress_entries[] = $entry;
7025		- }
7026		- }
7027		-
7028		- // =============================================
7029		- // PHASE 2: Single batch delete to Pinecone
7030		- // =============================================
7031		- if (!empty($all_vector_ids)) {
7032		- $all_vector_ids = array_values(array_unique($all_vector_ids));
7033		- $pinecone_success = true;
7034		- $batches = array_chunk($all_vector_ids, 100);
7035		-
7036		- foreach ($batches as $batch) {
7037		- $delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
7038		- 'headers' => array(
7039		- 'Api-Key' => $api_key,
7040		- 'accept' => 'application/json',
7041		- 'content-type' => 'application/json'
7042		- ),
7043		- 'body' => wp_json_encode(array('ids' => $batch)),
7044		- 'timeout' => 60
7045		- ));
7046		-
7047		- if (is_wp_error($delete_response)) {
7048		- $pinecone_success = false;
7049		- $errors[] = 'Pinecone batch deletion failed: ' . $delete_response->get_error_message();
7050		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed: ' . $delete_response->get_error_message());
7051		- } else {
7052		- $response_code = wp_remote_retrieve_response_code($delete_response);
7053		- if ($response_code !== 200) {
7054		- $pinecone_success = false;
7055		- $response_body = wp_remote_retrieve_body($delete_response);
7056		- $errors[] = "Pinecone API error (HTTP $response_code)";
7057		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed (HTTP ' . $response_code . ')', array('response' => substr($response_body, 0, 200)));
7058		- }
7059		- }
7060		- }
7061		-
7062		- // Mark all pinecone entries based on batch result
7063		- foreach ($pinecone_entry_ids as $eid) {
7064		- if ($pinecone_success) {
7065		- $success_ids[] = $eid;
7066		- } else {
7067		- $failed_ids[] = $eid;
7068		- }
7069		- }
7070		- }
7071		-
7072		- // =============================================
7073		- // PHASE 3: WordPress database deletions
7074		- // =============================================
7075		- foreach ($wordpress_entries as $entry) {
7076		- $entry_id = sanitize_text_field($entry['id'] ?? '');
7077		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
7078		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
7079		-
7080		- if (empty($entry_id)) {
7081		- continue;
7082		- }
7083		-
7084		- try {
7085		- if ($is_group && !empty($source_url)) {
7086		- $result = $wpdb->delete(
7087		- $table_name,
7088		- array('source_url' => $source_url),
7089		- array('%s')
7090		- );
7091		- } else {
7092		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
7093		- $result = $wpdb->delete(
7094		- $table_name,
7095		- array('id' => intval($entry_id)),
7096		- array('%d')
7097		- );
7098		- }
7099		-
7100		- if ($result !== false) {
7101		- $success_ids[] = $entry_id;
7102		- } else {
7103		- $failed_ids[] = $entry_id;
7104		- $errors[] = "Database error for entry: $entry_id";
7105		- }
7106		- } catch (Exception $e) {
7107		- $failed_ids[] = $entry_id;
7108		- $errors[] = $e->getMessage();
7109		- }
7110		- }
7111		-
7112		- wp_send_json_success(array(
7113		- 'success_ids' => $success_ids,
7114		- 'failed_ids' => $failed_ids,
7115		- 'errors' => $errors,
7116		- 'total_processed' => count($success_ids) + count($failed_ids)
7117		- ));
7118		-
7119		- exit;
7120		-}
7121		-
7122		-/**
7123		- * Get hierarchical roles for dropdown
7124		- */
7125		-public function mxchat_get_role_options() {
7126		- return array(
7127		- 'public' => __('Public (Everyone)', 'mxchat'),
7128		- 'logged_in' => __('Logged In Users', 'mxchat'),
7129		- 'subscriber' => __('Subscribers & Above', 'mxchat'),
7130		- 'contributor' => __('Contributors & Above', 'mxchat'),
7131		- 'author' => __('Authors & Above', 'mxchat'),
7132		- 'editor' => __('Editors & Above', 'mxchat'),
7133		- 'administrator' => __('Administrators Only', 'mxchat')
7134		- );
7135		-}
7136		-
7137		-/**
7138		- * Check if user has access to content based on role restriction
7139		- */
7140		-public function mxchat_user_has_content_access($role_restriction) {
7141		- // Public content is always accessible
7142		- if ($role_restriction === 'public' || empty($role_restriction)) {
7143		- return true;
7144		- }
7145	3484
7146		- // Check if user is logged in for logged_in restriction
7147		- if ($role_restriction === 'logged_in') {
7148		- return is_user_logged_in();
7149		- }
7150		-
7151		- // If not logged in, no access to role-restricted content
7152		- if (!is_user_logged_in()) {
7153		- return false;
7154		- }
7155		-
7156		- $user = wp_get_current_user();
7157		- $user_roles = $user->roles;
7158		-
7159		- if (empty($user_roles)) {
7160		- return false;
7161		- }
7162		-
7163		- // Define role hierarchy (higher number = higher access)
7164		- $hierarchy = array(
7165		- 'subscriber' => 1,
7166		- 'contributor' => 2,
7167		- 'author' => 3,
7168		- 'editor' => 4,
7169		- 'administrator' => 5
7170		- );
7171		-
7172		- // Get required level
7173		- $required_level = isset($hierarchy[$role_restriction]) ? $hierarchy[$role_restriction] : 0;
7174		-
7175		- // Check if user has required level or higher
7176		- foreach ($user_roles as $user_role) {
7177		- $user_level = isset($hierarchy[$user_role]) ? $hierarchy[$user_role] : 0;
7178		- if ($user_level >= $required_level) {
7179		- return true;
7180		- }
7181		- }
7182		-
7183		- return false;
7184		-}
7185		-
7186		-/**
7187		- * Handle role restriction updates via AJAX
7188		- * Removed cache clearing call since we removed caching
7189		- */
7190		-public function ajax_mxchat_update_role_restriction() {
7191		- // Verify nonce and permissions
7192		- if (!check_ajax_referer('mxchat_update_role_nonce', 'nonce', false)) {
7193		- wp_send_json_error('Invalid nonce');
7194		- exit;
7195		- }
7196		-
7197		- if (!current_user_can('manage_options')) {
7198		- wp_send_json_error('Unauthorized access');
7199		- exit;
7200		- }
7201		-
7202		- $entry_id = isset($_POST['entry_id']) ? sanitize_text_field($_POST['entry_id']) : '';
7203		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
7204		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
7205		-
7206		- if (empty($entry_id)) {
7207		- wp_send_json_error('Invalid entry ID');
7208		- exit;
7209		- }
7210		-
7211		- // Get knowledge manager instance to validate role restriction
7212		- $knowledge_manager = MxChat_Knowledge_Manager::get_instance();
7213		- $valid_roles = array_keys($knowledge_manager->mxchat_get_role_options());
7214		- if (!in_array($role_restriction, $valid_roles)) {
7215		- wp_send_json_error('Invalid role restriction');
7216		- exit;
7217		- }
7218		-
7219		- global $wpdb;
7220		-
7221		- if ($data_source === 'pinecone') {
7222		- // Handle Pinecone role restriction (stored separately in WordPress table)
7223		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7224		-
7225		- // Use REPLACE to insert or update the role restriction
7226		- $result = $wpdb->replace(
7227		- $roles_table,
7228		- array(
7229		- 'vector_id' => $entry_id,
7230		- 'role_restriction' => $role_restriction,
7231		- 'updated_at' => current_time('mysql')
7232		- ),
7233		- array('%s', '%s', '%s')
7234		- );
7235		-
7236		- // No cache clearing needed since we removed caching
7237		-
7238		- } else {
7239		- // Handle WordPress database role restriction (existing functionality)
7240		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7241		-
7242		- $result = $wpdb->update(
7243		- $table_name,
7244		- array('role_restriction' => $role_restriction),
7245		- array('id' => absint($entry_id)),
7246		- array('%s'),
7247		- array('%d')
7248		- );
7249		- }
7250		-
7251		- if ($result === false) {
7252		- wp_send_json_error('Database update failed: ' . $wpdb->last_error);
7253		- exit;
7254		- }
7255		-
7256		- wp_send_json_success(array(
7257		- 'message' => 'Role restriction updated successfully',
7258		- 'role_restriction' => $role_restriction,
7259		- 'data_source' => $data_source,
7260		- 'entry_id' => $entry_id
7261		- ));
7262		- exit;
7263		-}
7264		-
7265		-// ========================================
7266		-// ROLE-BASED CONTENT RESTRICTIONS - NEW FUNCTIONS
7267		-// Add these to your MxChat_Knowledge_Manager class
7268		-// ========================================
7269		-
7270		-/**
7271		- * Initialize role-based content hooks
7272		- * Add this call to your __construct() or mxchat_init_hooks() method
7273		- */
7274		-private function mxchat_init_role_hooks() {
7275		- // AJAX handlers for tag-role mappings
7276		- add_action('wp_ajax_mxchat_add_tag_role_mapping', array($this, 'ajax_add_tag_role_mapping'));
7277		- add_action('wp_ajax_mxchat_delete_tag_role_mapping', array($this, 'ajax_delete_tag_role_mapping'));
7278		- add_action('wp_ajax_mxchat_get_tag_role_mappings', array($this, 'ajax_get_tag_role_mappings'));
7279		- add_action('wp_ajax_mxchat_bulk_update_tag_roles', array($this, 'ajax_bulk_update_tag_roles'));
7280		-
7281		- // Hook to automatically update role restrictions when tags are added/removed
7282		- add_action('set_object_terms', array($this, 'handle_tag_change'), 10, 6);
7283		-
7284		- // Hook to apply role restrictions on auto-sync
7285		- add_action('mxchat_content_stored', array($this, 'apply_role_restriction_after_storage'), 10, 2);
7286		-}
7287		-
7288		-/**
7289		- * Add tag-role mapping via AJAX
7290		- */
7291		-public function ajax_add_tag_role_mapping() {
7292		- // Verify nonce and permissions
7293		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7294		-
7295		- if (!current_user_can('manage_options')) {
7296		- wp_send_json_error('Unauthorized access');
7297		- exit;
7298		- }
7299		-
7300		- $tag_input = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
7301		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
7302		-
7303		- if (empty($tag_input)) {
7304		- wp_send_json_error('Please enter a tag name or slug');
7305		- exit;
7306		- }
7307		-
7308		- // Validate role restriction
7309		- $valid_roles = array_keys($this->mxchat_get_role_options());
7310		- if (!in_array($role_restriction, $valid_roles)) {
7311		- wp_send_json_error('Invalid role restriction');
7312		- exit;
7313		- }
7314		-
7315		- // Resolve the tag by slug first, then fall back to its display name, so users can
7316		- // enter either "premium-content" or "Premium Content". (plan b8bcf5 — the field is
7317		- // labeled by name but previously validated by slug only, producing the confusing
7318		- // "Tag does not exist in WordPress" error when a real tag's name was typed.)
7319		- $term = get_term_by('slug', $tag_input, 'post_tag');
7320		- if (!$term) {
7321		- $term = get_term_by('name', $tag_input, 'post_tag');
7322		- }
7323		- if (!$term) {
7324		- wp_send_json_error('No tag with that name or slug exists yet. Create it under Posts → Tags first, then enter its name or slug.');
7325		- exit;
7326		- }
7327		-
7328		- // Always key the mapping by the RESOLVED slug — apply_role_restriction_to_post()
7329		- // compares against each post's tag slugs, so the stored key must be a slug,
7330		- // never the raw (possibly display-name) input.
7331		- $tag_slug = $term->slug;
7332		-
7333		- // Get existing mappings
7334		- $mappings = get_option('mxchat_tag_role_mappings', array());
7335		-
7336		- // Check if mapping already exists
7337		- if (isset($mappings[$tag_slug])) {
7338		- wp_send_json_error('Mapping for this tag already exists');
7339		- exit;
7340		- }
7341		-
7342		- // Add new mapping
7343		- $mappings[$tag_slug] = $role_restriction;
7344		- update_option('mxchat_tag_role_mappings', $mappings);
7345		-
7346		- wp_send_json_success(array(
7347		- 'message' => 'Tag-role mapping added successfully',
7348		- 'tag_slug' => $tag_slug,
7349		- 'role_restriction' => $role_restriction
7350		- ));
7351		- exit;
7352		-}
7353		-
7354		-/**
7355		- * Delete tag-role mapping via AJAX
7356		- */
7357		-public function ajax_delete_tag_role_mapping() {
7358		- // Verify nonce and permissions
7359		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7360		-
7361		- if (!current_user_can('manage_options')) {
7362		- wp_send_json_error('Unauthorized access');
7363		- exit;
7364		- }
7365		-
7366		- $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
7367		-
7368		- if (empty($tag_slug)) {
7369		- wp_send_json_error('Tag slug is required');
7370		- exit;
7371		- }
7372		-
7373		- // Get existing mappings
7374		- $mappings = get_option('mxchat_tag_role_mappings', array());
7375		-
7376		- // Check if mapping exists
7377		- if (!isset($mappings[$tag_slug])) {
7378		- wp_send_json_error('Mapping does not exist');
7379		- exit;
7380		- }
7381		-
7382		- // Remove mapping
7383		- unset($mappings[$tag_slug]);
7384		- update_option('mxchat_tag_role_mappings', $mappings);
7385		-
7386		- wp_send_json_success(array(
7387		- 'message' => 'Tag-role mapping deleted successfully',
7388		- 'tag_slug' => $tag_slug
7389		- ));
7390		- exit;
7391		-}
7392		-
7393		-/**
7394		- * Get all tag-role mappings via AJAX
7395		- */
7396		-public function ajax_get_tag_role_mappings() {
7397		- // Verify nonce and permissions
7398		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7399		-
7400		- if (!current_user_can('manage_options')) {
7401		- wp_send_json_error('Unauthorized access');
7402		- exit;
7403		- }
7404		-
7405		- // Get mappings
7406		- $mappings = get_option('mxchat_tag_role_mappings', array());
7407		- $role_options = $this->mxchat_get_role_options();
7408		-
7409		- $formatted_mappings = array();
7410		-
7411		- foreach ($mappings as $tag_slug => $role_restriction) {
7412		- // Get tag object
7413		- $term = get_term_by('slug', $tag_slug, 'post_tag');
7414		-
7415		- // Count posts with this tag
7416		- $post_count = 0;
7417		- if ($term) {
7418		- $post_count = $term->count;
7419		- }
7420		-
7421		- $formatted_mappings[] = array(
7422		- 'tag_slug' => $tag_slug,
7423		- 'role_restriction' => $role_restriction,
7424		- 'role_label' => $role_options[$role_restriction] ?? $role_restriction,
7425		- 'post_count' => $post_count
7426		- );
7427		- }
7428		-
7429		- wp_send_json_success(array(
7430		- 'mappings' => $formatted_mappings
7431		- ));
7432		- exit;
7433		-}
7434		-
7435		-/**
7436		- * Bulk update role restrictions for all existing content with mapped tags
7437		- */
7438		-public function ajax_bulk_update_tag_roles() {
7439		- // Verify nonce and permissions
7440		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7441		-
7442		- if (!current_user_can('manage_options')) {
7443		- wp_send_json_error('Unauthorized access');
7444		- exit;
7445		- }
7446		-
7447		- // Get mappings
7448		- $mappings = get_option('mxchat_tag_role_mappings', array());
7449		-
7450		- if (empty($mappings)) {
7451		- wp_send_json_error('No tag-role mappings found');
7452		- exit;
7453		- }
7454		-
7455		- global $wpdb;
7456		-
7457		- // Check if using Pinecone
7458		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
7459		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7460		-
7461		- $updated_count = 0;
7462		- $details = array();
7463		-
7464		- foreach ($mappings as $tag_slug => $role_restriction) {
7465		- // Get all posts with this tag
7466		- $posts = get_posts(array(
7467		- 'tag' => $tag_slug,
7468		- 'post_type' => 'any',
7469		- 'posts_per_page' => -1,
7470		- 'fields' => 'ids',
7471		- 'post_status' => 'publish'
7472		- ));
7473		-
7474		- if (empty($posts)) {
7475		- continue;
7476		- }
7477		-
7478		- $tag_updated = 0;
7479		-
7480		- foreach ($posts as $post_id) {
7481		- $source_url = get_permalink($post_id);
7482		- if (!$source_url) {
7483		- continue;
7484		- }
7485		-
7486		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
7487		- // Update Pinecone role restriction
7488		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7489		- $vector_id = md5($source_url);
7490		-
7491		- $result = $wpdb->replace(
7492		- $roles_table,
7493		- array(
7494		- 'vector_id' => $vector_id,
7495		- 'role_restriction' => $role_restriction,
7496		- 'updated_at' => current_time('mysql')
7497		- ),
7498		- array('%s', '%s', '%s')
7499		- );
7500		- } else {
7501		- // Update WordPress DB
7502		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7503		-
7504		- $result = $wpdb->update(
7505		- $table_name,
7506		- array('role_restriction' => $role_restriction),
7507		- array('source_url' => $source_url),
7508		- array('%s'),
7509		- array('%s')
7510		- );
7511		- }
7512		-
7513		- if ($result !== false) {
7514		- $tag_updated++;
7515		- $updated_count++;
7516		- }
7517		- }
7518		-
7519		- if ($tag_updated > 0) {
7520		- $details[] = sprintf(
7521		- 'Tag "%s" (%s): %d posts updated',
7522		- $tag_slug,
7523		- $role_restriction,
7524		- $tag_updated
7525		- );
7526		- }
7527		- }
7528		-
7529		- wp_send_json_success(array(
7530		- 'message' => 'Bulk update completed',
7531		- 'updated_count' => $updated_count,
7532		- 'tags_processed' => count($mappings),
7533		- 'details' => $details
7534		- ));
7535		- exit;
7536		-}
7537		-
7538		-/**
7539		- * Handle tag changes on posts (when tags are added or removed)
7540		- */
7541		-public function handle_tag_change($object_id, $terms, $tt_ids, $taxonomy, $append, $old_tt_ids) {
7542		- // Only process post tags
7543		- if ($taxonomy !== 'post_tag') {
7544		- return;
7545		- }
7546		-
7547		- // Get tag-role mappings
7548		- $mappings = get_option('mxchat_tag_role_mappings', array());
7549		-
7550		- if (empty($mappings)) {
7551		- return;
7552		- }
7553		-
7554		- // Get the post's URL
7555		- $source_url = get_permalink($object_id);
7556		- if (!$source_url) {
7557		- return;
7558		- }
7559		-
7560		- // Determine the highest role restriction based on tags
7561		- $highest_role = 'public';
7562		- $role_hierarchy = array(
7563		- 'public' => 0,
7564		- 'logged_in' => 1,
7565		- 'subscriber' => 2,
7566		- 'contributor' => 3,
7567		- 'author' => 4,
7568		- 'editor' => 5,
7569		- 'administrator' => 6
7570		- );
7571		-
7572		- // Get all current tags for the post
7573		- $current_tags = wp_get_post_tags($object_id, array('fields' => 'slugs'));
7574		-
7575		- // Find the highest role restriction among the tags
7576		- foreach ($current_tags as $tag_slug) {
7577		- if (isset($mappings[$tag_slug])) {
7578		- $role = $mappings[$tag_slug];
7579		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
7580		- $highest_role = $role;
7581		- }
7582		- }
7583		- }
7584		-
7585		- // Update the role restriction in the database
7586		- global $wpdb;
7587		-
7588		- // Check if using Pinecone
7589		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
7590		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7591		-
7592		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
7593		- // Update Pinecone role restriction
7594		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7595		- $vector_id = md5($source_url);
7596		-
7597		- $wpdb->replace(
7598		- $roles_table,
7599		- array(
7600		- 'vector_id' => $vector_id,
7601		- 'role_restriction' => $highest_role,
7602		- 'updated_at' => current_time('mysql')
7603		- ),
7604		- array('%s', '%s', '%s')
7605		- );
7606		- } else {
7607		- // Update WordPress DB
7608		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7609		-
7610		- $wpdb->update(
7611		- $table_name,
7612		- array('role_restriction' => $highest_role),
7613		- array('source_url' => $source_url),
7614		- array('%s'),
7615		- array('%s')
7616		- );
7617		- }
7618		-}
7619		-
7620		-/**
7621		- * Apply role restriction after content is stored (for auto-sync)
7622		- */
7623		-public function apply_role_restriction_after_storage($post_id, $source_url) {
7624		- // Get tag-role mappings
7625		- $mappings = get_option('mxchat_tag_role_mappings', array());
7626		-
7627		- if (empty($mappings)) {
7628		- return;
7629		- }
7630		-
7631		- // Get all tags for the post
7632		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
7633		-
7634		- if (empty($post_tags)) {
7635		- return;
7636		- }
7637		-
7638		- // Determine the highest role restriction based on tags
7639		- $highest_role = 'public';
7640		- $role_hierarchy = array(
7641		- 'public' => 0,
7642		- 'logged_in' => 1,
7643		- 'subscriber' => 2,
7644		- 'contributor' => 3,
7645		- 'author' => 4,
7646		- 'editor' => 5,
7647		- 'administrator' => 6
7648		- );
7649		-
7650		- foreach ($post_tags as $tag_slug) {
7651		- if (isset($mappings[$tag_slug])) {
7652		- $role = $mappings[$tag_slug];
7653		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
7654		- $highest_role = $role;
7655		- }
7656		- }
7657		- }
7658		-
7659		- // If no restricted tags found, return (leave as public)
7660		- if ($highest_role === 'public') {
7661		- return;
7662		- }
7663		-
7664		- // Update the role restriction
7665		- global $wpdb;
7666		-
7667		- // Check if using Pinecone
7668		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
7669		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7670		-
7671		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
7672		- // Update Pinecone role restriction
7673		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7674		- $vector_id = md5($source_url);
7675		-
7676		- $wpdb->replace(
7677		- $roles_table,
7678		- array(
7679		- 'vector_id' => $vector_id,
7680		- 'role_restriction' => $highest_role,
7681		- 'updated_at' => current_time('mysql')
7682		- ),
7683		- array('%s', '%s', '%s')
7684		- );
7685		- } else {
7686		- // Update WordPress DB
7687		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7688		-
7689		- $wpdb->update(
7690		- $table_name,
7691		- array('role_restriction' => $highest_role),
7692		- array('source_url' => $source_url),
7693		- array('%s'),
7694		- array('%s')
7695		- );
7696		- }
7697		-}
7698		-
7699		-
7700	3485	// ========================================
7701	3486	// HELPER METHODS
7702	3487	// ========================================
7703	3488
		@@ -7764,1062 +3549,8 @@
7764	3549	*/
7765	3550	private function mxchat_get_pinecone_manager() {
7766	3551	return MxChat_Pinecone_Manager::get_instance();
7767	3552	}
7768		-
7769		-
7770		- // ========================================
7771		-// DATABASE QUEUE TABLE MANAGEMENT
7772		-// ========================================
7773		-
7774		-/**
7775		- * Create queue table on plugin activation
7776		- * Call this from your plugin activation hook
7777		- */
7778		-public function mxchat_create_queue_table() {
7779		- global $wpdb;
7780		-
7781		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7782		- $charset_collate = $wpdb->get_charset_collate();
7783		-
7784		- $sql = "CREATE TABLE IF NOT EXISTS $table_name (
7785		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
7786		- queue_id varchar(64) NOT NULL,
7787		- item_type varchar(20) NOT NULL,
7788		- item_data longtext NOT NULL,
7789		- status varchar(20) NOT NULL DEFAULT 'pending',
7790		- bot_id varchar(50) NOT NULL DEFAULT 'default',
7791		- priority int(11) NOT NULL DEFAULT 0,
7792		- attempts int(11) NOT NULL DEFAULT 0,
7793		- max_attempts int(11) NOT NULL DEFAULT 3,
7794		- error_message text DEFAULT NULL,
7795		- created_at datetime NOT NULL,
7796		- started_at datetime DEFAULT NULL,
7797		- completed_at datetime DEFAULT NULL,
7798		- PRIMARY KEY (id),
7799		- KEY queue_id (queue_id),
7800		- KEY status (status),
7801		- KEY item_type (item_type),
7802		- KEY priority (priority)
7803		- ) $charset_collate;";
7804		-
7805		- require_once(ABSPATH . 'wp-admin/includes/upgrade.php');
7806		- dbDelta($sql);
7807		-
7808		- // Also create a meta table for queue metadata
7809		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7810		-
7811		- $meta_sql = "CREATE TABLE IF NOT EXISTS $meta_table (
7812		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
7813		- queue_id varchar(64) NOT NULL,
7814		- meta_key varchar(255) NOT NULL,
7815		- meta_value longtext,
7816		- PRIMARY KEY (id),
7817		- KEY queue_id (queue_id),
7818		- KEY meta_key (meta_key)
7819		- ) $charset_collate;";
7820		-
7821		- dbDelta($meta_sql);
7822		-}
7823		-
7824		-/**
7825		- * Add items to the processing queue
7826		- *
7827		- * @param string $queue_id Unique identifier for this queue batch
7828		- * @param string $item_type Type of item (url, pdf_page)
7829		- * @param array $items Array of items to queue
7830		- * @param string $bot_id Bot ID for processing
7831		- * @return int Number of items queued
7832		- */
7833		-private function mxchat_add_to_queue($queue_id, $item_type, $items, $bot_id = 'default') {
7834		- global $wpdb;
7835		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7836		-
7837		- $queued_count = 0;
7838		- $priority = 0;
7839		-
7840		- foreach ($items as $item) {
7841		- $result = $wpdb->insert(
7842		- $table_name,
7843		- array(
7844		- 'queue_id' => $queue_id,
7845		- 'item_type' => $item_type,
7846		- 'item_data' => wp_json_encode($item),
7847		- 'status' => 'pending',
7848		- 'bot_id' => $bot_id,
7849		- 'priority' => $priority,
7850		- 'attempts' => 0,
7851		- 'max_attempts' => 3,
7852		- 'created_at' => current_time('mysql')
7853		- ),
7854		- array('%s', '%s', '%s', '%s', '%s', '%d', '%d', '%d', '%s')
7855		- );
7856		-
7857		- if ($result) {
7858		- $queued_count++;
7859		- }
7860		-
7861		- $priority++; // Process in order
7862		- }
7863		-
7864		- return $queued_count;
7865		-}
7866		-
7867		-/**
7868		- * Store queue metadata (total counts, source URL, etc.)
7869		- */
7870		-private function mxchat_set_queue_meta($queue_id, $meta_key, $meta_value) {
7871		- global $wpdb;
7872		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7873		-
7874		- // Check if meta exists
7875		- $existing = $wpdb->get_var($wpdb->prepare(
7876		- "SELECT id FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
7877		- $queue_id,
7878		- $meta_key
7879		- ));
7880		-
7881		- if ($existing) {
7882		- // Update
7883		- $wpdb->update(
7884		- $meta_table,
7885		- array('meta_value' => maybe_serialize($meta_value)),
7886		- array('queue_id' => $queue_id, 'meta_key' => $meta_key),
7887		- array('%s'),
7888		- array('%s', '%s')
7889		- );
7890		- } else {
7891		- // Insert
7892		- $wpdb->insert(
7893		- $meta_table,
7894		- array(
7895		- 'queue_id' => $queue_id,
7896		- 'meta_key' => $meta_key,
7897		- 'meta_value' => maybe_serialize($meta_value)
7898		- ),
7899		- array('%s', '%s', '%s')
7900		- );
7901		- }
7902		-}
7903		-
7904		-/**
7905		- * Get queue metadata
7906		- */
7907		-private function mxchat_get_queue_meta($queue_id, $meta_key) {
7908		- global $wpdb;
7909		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7910		-
7911		- $value = $wpdb->get_var($wpdb->prepare(
7912		- "SELECT meta_value FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
7913		- $queue_id,
7914		- $meta_key
7915		- ));
7916		-
7917		- return maybe_unserialize($value);
7918		-}
7919		-
7920		-// ========================================
7921		-// AJAX QUEUE PROCESSING HANDLERS
7922		-// ========================================
7923		-
7924		-/**
7925		- * AJAX: Get next item from queue to process
7926		- */
7927		-public function ajax_mxchat_get_next_queue_item() {
7928		- // Verify nonce and permissions
7929		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7930		-
7931		- if (!current_user_can('manage_options')) {
7932		- wp_send_json_error('Unauthorized access');
7933		- }
7934		-
7935		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7936		-
7937		- if (empty($queue_id)) {
7938		- wp_send_json_error('Missing queue ID');
7939		- }
7940		-
7941		- global $wpdb;
7942		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7943		-
7944		- // Get next pending item with retry logic for failed items
7945		- $next_item = $wpdb->get_row($wpdb->prepare(
7946		- "SELECT * FROM $table_name
7947		- WHERE queue_id = %s
7948		- AND status IN ('pending', 'failed')
7949		- AND attempts < max_attempts
7950		- ORDER BY priority ASC, id ASC
7951		- LIMIT 1",
7952		- $queue_id
7953		- ));
7954		-
7955		- if (!$next_item) {
7956		- // No more items - queue complete
7957		- wp_send_json_success(array(
7958		- 'complete' => true,
7959		- 'message' => 'Queue processing complete'
7960		- ));
7961		- }
7962		-
7963		- // Mark item as processing
7964		- $wpdb->update(
7965		- $table_name,
7966		- array(
7967		- 'status' => 'processing',
7968		- 'started_at' => current_time('mysql'),
7969		- 'attempts' => $next_item->attempts + 1
7970		- ),
7971		- array('id' => $next_item->id),
7972		- array('%s', '%s', '%d'),
7973		- array('%d')
7974		- );
7975		-
7976		- wp_send_json_success(array(
7977		- 'complete' => false,
7978		- 'item' => array(
7979		- 'id' => $next_item->id,
7980		- 'type' => $next_item->item_type,
7981		- 'data' => json_decode($next_item->item_data, true),
7982		- 'bot_id' => $next_item->bot_id,
7983		- 'attempt' => $next_item->attempts + 1
7984		- )
7985		- ));
7986		-}
7987		-
7988		-/**
7989		- * AJAX: Process a single queue item
7990		- */
7991		-public function ajax_mxchat_process_queue_item() {
7992		- // Verify nonce and permissions
7993		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7994		-
7995		- if (!current_user_can('manage_options')) {
7996		- wp_send_json_error('Unauthorized access');
7997		- }
7998		-
7999		- $item_id = isset($_POST['item_id']) ? absint($_POST['item_id']) : 0;
8000		- $item_type = isset($_POST['item_type']) ? sanitize_text_field($_POST['item_type']) : '';
8001		- $item_data = isset($_POST['item_data']) ? $_POST['item_data'] : array();
8002		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
8003		-
8004		- if (empty($item_id) || empty($item_type)) {
8005		- wp_send_json_error('Missing item data');
8006		- }
8007		-
8008		- global $wpdb;
8009		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8010		-
8011		- // Process based on item type
8012		- try {
8013		- set_time_limit(60); // Give processing 60 seconds
8014		-
8015		- $result = false;
8016		- $error_message = '';
8017		-
8018		- // Read item directly from DB to get queue_id and preserve special chars in item_data
8019		- // (POST round-trip through JS mangles characters like apostrophes in URLs)
8020		- $db_item = $wpdb->get_row($wpdb->prepare(
8021		- "SELECT queue_id, item_data FROM $table_name WHERE id = %d",
8022		- $item_id
8023		- ));
8024		- $item_queue_id = $db_item ? $db_item->queue_id : '';
8025		- if ($db_item && !empty($db_item->item_data)) {
8026		- $db_data = json_decode($db_item->item_data, true);
8027		- if (is_array($db_data)) {
8028		- $item_data = $db_data;
8029		- }
8030		- }
8031		-
8032		- switch ($item_type) {
8033		- case 'url':
8034		- $result = $this->mxchat_process_queue_url($item_data, $bot_id, $item_queue_id);
8035		- break;
8036		-
8037		- case 'pdf_page':
8038		- $result = $this->mxchat_process_queue_pdf_page($item_data, $bot_id);
8039		- break;
8040		-
8041		- default:
8042		- throw new Exception('Unknown item type: ' . $item_type);
8043		- }
8044		-
8045		- if (is_wp_error($result)) {
8046		- $error_code = $result->get_error_code();
8047		- // Content errors (empty page, sanitization) are permanent — retrying won't help
8048		- $permanent_codes = array('empty_page', 'empty_after_sanitization', 'no_api_key', 'page_not_found');
8049		- if (in_array($error_code, $permanent_codes)) {
8050		- // Mark as permanently failed — set attempts = max_attempts so it won't be retried
8051		- $current_item = $wpdb->get_row($wpdb->prepare(
8052		- "SELECT max_attempts FROM $table_name WHERE id = %d", $item_id
8053		- ));
8054		- $wpdb->update(
8055		- $table_name,
8056		- array(
8057		- 'status' => 'failed',
8058		- 'error_message' => $result->get_error_message(),
8059		- 'attempts' => $current_item ? $current_item->max_attempts : 3
8060		- ),
8061		- array('id' => $item_id),
8062		- array('%s', '%s', '%d'),
8063		- array('%d')
8064		- );
8065		- wp_send_json_error(array(
8066		- 'message' => $result->get_error_message(),
8067		- 'permanent_failure' => true,
8068		- 'item_id' => $item_id
8069		- ));
8070		- return;
8071		- }
8072		- throw new Exception($result->get_error_message());
8073		- }
8074		-
8075		- if ($result === false) {
8076		- throw new Exception('Processing returned false - item may be empty or invalid');
8077		- }
8078		-
8079		- // Mark as completed
8080		- $wpdb->update(
8081		- $table_name,
8082		- array(
8083		- 'status' => 'completed',
8084		- 'completed_at' => current_time('mysql'),
8085		- 'error_message' => null
8086		- ),
8087		- array('id' => $item_id),
8088		- array('%s', '%s', '%s'),
8089		- array('%d')
8090		- );
8091		-
8092		- wp_send_json_success(array(
8093		- 'processed' => true,
8094		- 'item_id' => $item_id,
8095		- 'message' => 'Item processed successfully'
8096		- ));
8097		-
8098		- } catch (Exception $e) {
8099		- $error_message = $e->getMessage();
8100		-
8101		- // Get current attempt count
8102		- $item = $wpdb->get_row($wpdb->prepare(
8103		- "SELECT attempts, max_attempts FROM $table_name WHERE id = %d",
8104		- $item_id
8105		- ));
8106		-
8107		- // Check if we've exhausted retries
8108		- if ($item && $item->attempts >= $item->max_attempts) {
8109		- // Permanently failed
8110		- $wpdb->update(
8111		- $table_name,
8112		- array(
8113		- 'status' => 'failed',
8114		- 'error_message' => $error_message
8115		- ),
8116		- array('id' => $item_id),
8117		- array('%s', '%s'),
8118		- array('%d')
8119		- );
8120		-
8121		- wp_send_json_error(array(
8122		- 'message' => 'Item failed after maximum attempts: ' . $error_message,
8123		- 'permanent_failure' => true,
8124		- 'item_id' => $item_id
8125		- ));
8126		- } else {
8127		- // Mark for retry
8128		- $wpdb->update(
8129		- $table_name,
8130		- array(
8131		- 'status' => 'failed',
8132		- 'error_message' => $error_message
8133		- ),
8134		- array('id' => $item_id),
8135		- array('%s', '%s'),
8136		- array('%d')
8137		- );
8138		-
8139		- wp_send_json_error(array(
8140		- 'message' => 'Item processing failed, will retry: ' . $error_message,
8141		- 'can_retry' => true,
8142		- 'item_id' => $item_id,
8143		- 'attempts' => $item ? $item->attempts : 0
8144		- ));
8145		- }
8146		- }
8147		-}
8148		-
8149		-/**
8150		- * Process a URL from the queue
8151		- */
8152		-private function mxchat_process_queue_url($item_data, $bot_id = 'default', $queue_id = '') {
8153		- $url = isset($item_data['url']) ? $item_data['url'] : '';
8154		-
8155		- if (empty($url)) {
8156		- return new WP_Error('invalid_url', 'URL is empty');
8157		- }
8158		-
8159		- // Get bot-specific embedding decision early (needed for both paths) —
8160		- // custom-provider-aware (plan cbd5fd). Error code preserved.
8161		- $bot_options = $this->get_bot_options($bot_id);
8162		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
8163		-
8164		- $preflight = MxChat_Utils::embedding_preflight($options);
8165		- if (!$preflight['ok']) {
8166		- return new WP_Error('no_api_key', $preflight['reason'] . ' (bot: ' . $bot_id . ')');
8167		- }
8168		- $api_key = $preflight['api_key'];
8169		-
8170		- // Check if this is a WooCommerce product URL and WooCommerce is active
8171		- $is_product_url = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
8172		- $content_type = $is_product_url ? 'product' : 'url';
8173		-
8174		- // Try to get WooCommerce product data if it's a product URL
8175		- if ($is_product_url && class_exists('WooCommerce')) {
8176		- $product_content = $this->mxchat_extract_woocommerce_product_content($url);
8177		-
8178		- if (!empty($product_content)) {
8179		- // Successfully extracted WooCommerce product data with pricing
8180		- $result = MxChat_Utils::submit_content_to_db(
8181		- $product_content,
8182		- $url,
8183		- $api_key,
8184		- null,
8185		- $bot_id,
8186		- 'product'
8187		- );
8188		- return $result;
8189		- }
8190		- // If WooCommerce extraction failed, fall through to HTML extraction
8191		- }
8192		-
8193		- // Fetch URL content (fallback for non-products or when WooCommerce extraction fails)
8194		- $is_likely_pdf = (strtolower(pathinfo(parse_url($url, PHP_URL_PATH) ?: '', PATHINFO_EXTENSION)) === 'pdf');
8195		- $response = wp_remote_get($url, array(
8196		- 'timeout' => $is_likely_pdf ? 120 : 30,
8197		- 'redirection' => 5,
8198		- 'user-agent' => mxchat_ingest_user_agent(),
8199		- ));
8200		-
8201		- if (is_wp_error($response)) {
8202		- return $response;
8203		- }
8204		-
8205		- $response_code = wp_remote_retrieve_response_code($response);
8206		- if ($response_code !== 200) {
8207		- return new WP_Error('http_error', 'HTTP ' . $response_code . ' error for: ' . $url);
8208		- }
8209		-
8210		- // Check if URL is a PDF — expand into per-page queue items using the standard PDF pipeline
8211		- if ($this->mxchat_is_pdf_url($url, $response)) {
8212		- return $this->mxchat_expand_pdf_to_queue($url, $response, $bot_id, $queue_id);
8213		- }
8214		-
8215		- $html = wp_remote_retrieve_body($response);
8216		-
8217		- if (empty($html)) {
8218		- return new WP_Error('empty_response', 'Empty response body');
8219		- }
8220		-
8221		- // Extract and sanitize content
8222		- $content = $this->mxchat_extract_main_content($html);
8223		- $sanitized = $this->mxchat_sanitize_content_for_api($content);
8224		-
8225		- if (empty($sanitized)) {
8226		- // Not an error - just no content found (maybe a redirect or empty page)
8227		- return false;
8228		- }
8229		-
8230		- // Submit to database with content_type
8231		- $result = MxChat_Utils::submit_content_to_db(
8232		- $sanitized,
8233		- $url,
8234		- $api_key,
8235		- null,
8236		- $bot_id,
8237		- $content_type
8238		- );
8239		-
8240		- return $result;
8241		-}
8242		-
8243		-/**
8244		- * Expand a PDF URL into per-page queue items using the standard PDF pipeline.
8245		- * Called when a sitemap URL turns out to be a PDF — downloads, parses page count,
8246		- * and adds pdf_page items to the same queue so they process with full progress tracking.
8247		- */
8248		-private function mxchat_expand_pdf_to_queue($pdf_url, $response, $bot_id = 'default', $queue_id = '') {
8249		- set_time_limit(120); // PDFs need extra time for download + parsing
8250		-
8251		- $upload_dir = wp_upload_dir();
8252		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
8253		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
8254		-
8255		- $response_body = wp_remote_retrieve_body($response);
8256		- if (empty($response_body)) {
8257		- return new WP_Error('empty_pdf', 'Empty PDF response for: ' . $pdf_url);
8258		- }
8259		-
8260		- if (!wp_mkdir_p(dirname($pdf_path))) {
8261		- return new WP_Error('dir_error', 'Failed to create upload directory');
8262		- }
8263		-
8264		- file_put_contents($pdf_path, $response_body);
8265		-
8266		- if (!file_exists($pdf_path)) {
8267		- return new WP_Error('save_error', 'Failed to save PDF file');
8268		- }
8269		-
8270		- try {
8271		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
8272		-
8273		- if ($total_pages === false || $total_pages < 1) {
8274		- wp_delete_file($pdf_path);
8275		- return new WP_Error('no_pages', 'PDF has no pages: ' . $pdf_url);
8276		- }
8277		-
8278		- // Build per-page items identical to mxchat_handle_pdf_for_knowledge_base
8279		- $pages = array();
8280		- for ($i = 1; $i <= $total_pages; $i++) {
8281		- $pages[] = array(
8282		- 'pdf_path' => $pdf_path,
8283		- 'pdf_url' => $pdf_url,
8284		- 'page_number' => $i,
8285		- 'total_pages' => $total_pages
8286		- );
8287		- }
8288		-
8289		- // Add pdf_page items to the SAME queue so the JS picks them up automatically
8290		- if (!empty($queue_id)) {
8291		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
8292		- } else {
8293		- // Fallback: create a new PDF queue (shouldn't happen in sitemap flow)
8294		- $new_queue_id = 'pdf_' . md5($pdf_url . time());
8295		- $queued_count = $this->mxchat_add_to_queue($new_queue_id, 'pdf_page', $pages, $bot_id);
8296		- $this->mxchat_set_queue_meta($new_queue_id, 'source_url', $pdf_url);
8297		- $this->mxchat_set_queue_meta($new_queue_id, 'queue_type', 'pdf');
8298		- $this->mxchat_set_queue_meta($new_queue_id, 'total_items', $total_pages);
8299		- $this->mxchat_set_queue_meta($new_queue_id, 'bot_id', $bot_id);
8300		- $this->mxchat_set_queue_meta($new_queue_id, 'pdf_path', $pdf_path);
8301		- $this->mxchat_set_queue_meta($new_queue_id, 'created_at', current_time('mysql'));
8302		- }
8303		-
8304		- if ($queued_count === 0) {
8305		- wp_delete_file($pdf_path);
8306		- return new WP_Error('queue_error', 'Failed to add PDF pages to queue');
8307		- }
8308		-
8309		- // Return true so the original URL item is marked complete
8310		- // The new pdf_page items will be processed in subsequent batches
8311		- return true;
8312		-
8313		- } catch (Exception $e) {
8314		- if (file_exists($pdf_path)) {
8315		- wp_delete_file($pdf_path);
8316		- }
8317		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
8318		- }
8319		-}
8320		-
8321		-/**
8322		- * Legacy: Process a PDF URL inline during sitemap queue processing.
8323		- * @deprecated Use mxchat_expand_pdf_to_queue instead — kept for reference only.
8324		- */
8325		-private function mxchat_process_pdf_url_inline($pdf_url, $response, $api_key, $bot_id = 'default') {
8326		- set_time_limit(120); // PDFs need more time — downloading + parsing all pages
8327		-
8328		- $upload_dir = wp_upload_dir();
8329		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
8330		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
8331		-
8332		- $response_body = wp_remote_retrieve_body($response);
8333		- if (empty($response_body)) {
8334		- return new WP_Error('empty_pdf', 'Empty PDF response');
8335		- }
8336		-
8337		- if (!wp_mkdir_p(dirname($pdf_path))) {
8338		- return new WP_Error('dir_error', 'Failed to create upload directory');
8339		- }
8340		-
8341		- file_put_contents($pdf_path, $response_body);
8342		-
8343		- if (!file_exists($pdf_path)) {
8344		- return new WP_Error('save_error', 'Failed to save PDF file');
8345		- }
8346		-
8347		- try {
8348		- mxchat_load_pdf_parser();
8349		- $parser = new \Smalot\PdfParser\Parser();
8350		- $pdf = $parser->parseFile($pdf_path);
8351		- $pages = $pdf->getPages();
8352		- $total_pages = count($pages);
8353		-
8354		- if ($total_pages < 1) {
8355		- wp_delete_file($pdf_path);
8356		- return new WP_Error('no_pages', 'PDF has no pages');
8357		- }
8358		-
8359		- $processed = 0;
8360		- $skipped_pages = array();
8361		-
8362		- for ($i = 0; $i < $total_pages; $i++) {
8363		- $page_num = $i + 1;
8364		- $text = $pages[$i]->getText();
8365		- if (empty($text)) {
8366		- $skipped_pages[] = 'Page ' . $page_num . ': No text could be extracted — page may contain only images, links, or non-standard encoding';
8367		- continue;
8368		- }
8369		-
8370		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
8371		- if (empty($sanitized)) {
8372		- $skipped_pages[] = 'Page ' . $page_num . ': Text was extracted but contained only special characters, control codes, or unsupported content';
8373		- continue;
8374		- }
8375		-
8376		- $metadata = array(
8377		- 'document_type' => 'pdf',
8378		- 'total_pages' => $total_pages,
8379		- 'current_page' => $page_num,
8380		- 'source_url' => $pdf_url,
8381		- );
8382		-
8383		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
8384		- $page_url = esc_url($pdf_url . '#page=' . $page_num);
8385		-
8386		- MxChat_Utils::submit_content_to_db(
8387		- $content_with_metadata,
8388		- $page_url,
8389		- $api_key,
8390		- null,
8391		- $bot_id,
8392		- 'pdf'
8393		- );
8394		-
8395		- $processed++;
8396		- }
8397		-
8398		- // Clean up the temp PDF file
8399		- wp_delete_file($pdf_path);
8400		-
8401		- if (!empty($skipped_pages)) {
8402		- error_log('MxChat PDF: Skipped ' . count($skipped_pages) . ' of ' . $total_pages . ' pages: ' . implode('; ', $skipped_pages));
8403		- }
8404		-
8405		- return $processed > 0 ? true : false;
8406		-
8407		- } catch (Exception $e) {
8408		- if (file_exists($pdf_path)) {
8409		- wp_delete_file($pdf_path);
8410		- }
8411		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
8412		- }
8413		-}
8414		-
8415		-/**
8416		- * Extract WooCommerce product content including pricing
8417		- *
8418		- * @param string $url The product URL
8419		- * @return string|false Product content with pricing, or false if not found
8420		- */
8421		-private function mxchat_extract_woocommerce_product_content($url) {
8422		- // Try to get product ID from URL
8423		- $product_id = url_to_postid($url);
8424		-
8425		- // If url_to_postid fails, try to extract from URL pattern
8426		- if (!$product_id) {
8427		- $product_slug = '';
8428		-
8429		- // Handle pretty permalinks: /product/product-name/
8430		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
8431		- $product_slug = $matches[1];
8432		- }
8433		-
8434		- if (!empty($product_slug)) {
8435		- $product_post = get_page_by_path($product_slug, OBJECT, 'product');
8436		- if ($product_post) {
8437		- $product_id = $product_post->ID;
8438		- }
8439		- }
8440		- }
8441		-
8442		- if (!$product_id) {
8443		- return false;
8444		- }
8445		-
8446		- // Get WooCommerce product object
8447		- $product = wc_get_product($product_id);
8448		-
8449		- if (!$product) {
8450		- return false;
8451		- }
8452		-
8453		- // Build product content with pricing (similar to mxchat_store_product_embedding)
8454		- $title = $product->get_name();
8455		- $description = $product->get_description();
8456		- $short_description = $product->get_short_description();
8457		- $sku = $product->get_sku();
8458		-
8459		- // Get pricing information
8460		- $regular_price = $product->get_regular_price();
8461		- $sale_price = $product->get_sale_price();
8462		- $price = $product->get_price(); // Current active price
8463		-
8464		- // Get currency symbol
8465		- $currency_symbol = get_woocommerce_currency_symbol();
8466		-
8467		- // Format content
8468		- $content = $title . "\n\n";
8469		-
8470		- if (!empty($short_description)) {
8471		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
8472		- }
8473		-
8474		- if (!empty($description)) {
8475		- $content .= wp_strip_all_tags($description) . "\n\n";
8476		- }
8477		-
8478		- // Add pricing information
8479		- if (!empty($regular_price)) {
8480		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
8481		- } elseif (!empty($price)) {
8482		- $content .= "Price: " . $currency_symbol . $price . "\n";
8483		- }
8484		-
8485		- if (!empty($sale_price) && $sale_price !== $regular_price) {
8486		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
8487		- }
8488		-
8489		- // Handle variable products - show price range
8490		- if ($product->is_type('variable')) {
8491		- $min_price = $product->get_variation_price('min');
8492		- $max_price = $product->get_variation_price('max');
8493		- if ($min_price !== $max_price) {
8494		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
8495		- }
8496		- }
8497		-
8498		- if (!empty($sku)) {
8499		- $content .= "SKU: " . $sku . "\n";
8500		- }
8501		-
8502		- // Get product categories
8503		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
8504		- if (!empty($categories) && !is_wp_error($categories)) {
8505		- $content .= "Categories: " . implode(', ', $categories) . "\n";
8506		- }
8507		-
8508		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
8509		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
8510		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
8511		- foreach ($custom_tabs as $tab) {
8512		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
8513		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
8514		-
8515		- if (!empty($tab_title) && !empty($tab_content)) {
8516		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
8517		- }
8518		- }
8519		- }
8520		-
8521		- // Also check for reusable/saved tabs applied to this product
8522		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
8523		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
8524		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
8525		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
8526		- foreach ($applied_saved_tabs as $saved_tab_id) {
8527		- if (isset($saved_tabs[$saved_tab_id])) {
8528		- $tab = $saved_tabs[$saved_tab_id];
8529		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
8530		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
8531		-
8532		- if (!empty($tab_title) && !empty($tab_content)) {
8533		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
8534		- }
8535		- }
8536		- }
8537		- }
8538		- }
8539		-
8540		- return $this->mxchat_sanitize_content_for_api($content);
8541		-}
8542		-
8543		-/**
8544		- * Process a PDF page from the queue
8545		- */
8546		-private function mxchat_process_queue_pdf_page($item_data, $bot_id = 'default') {
8547		- $pdf_path = isset($item_data['pdf_path']) ? $item_data['pdf_path'] : '';
8548		- $pdf_url = isset($item_data['pdf_url']) ? $item_data['pdf_url'] : '';
8549		- $page_number = isset($item_data['page_number']) ? absint($item_data['page_number']) : 0;
8550		- $total_pages = isset($item_data['total_pages']) ? absint($item_data['total_pages']) : 0;
8551		-
8552		- if (empty($pdf_path) || !file_exists($pdf_path)) {
8553		- return new WP_Error('pdf_not_found', 'PDF file not found: ' . $pdf_path);
8554		- }
8555		-
8556		- if ($page_number < 1) {
8557		- return new WP_Error('invalid_page', 'Invalid page number');
8558		- }
8559		-
8560		- try {
8561		- mxchat_load_pdf_parser();
8562		- $parser = new \Smalot\PdfParser\Parser();
8563		- $pdf = $parser->parseFile($pdf_path);
8564		- $pages = $pdf->getPages();
8565		-
8566		- if (!isset($pages[$page_number - 1])) {
8567		- return new WP_Error('page_not_found', 'Page ' . $page_number . ' not found in PDF');
8568		- }
8569		-
8570		- $text = $pages[$page_number - 1]->getText();
8571		-
8572		- if (empty($text)) {
8573		- return new WP_Error('empty_page', 'Page ' . $page_number . ': No text could be extracted — page may contain only images, links, or non-standard encoding');
8574		- }
8575		-
8576		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
8577		-
8578		- if (empty($sanitized)) {
8579		- return new WP_Error('empty_after_sanitization', 'Page ' . $page_number . ': Text was extracted but contained only special characters, control codes, or unsupported content that was removed during cleanup');
8580		- }
8581		-
8582		- // Create metadata
8583		- $metadata = array(
8584		- 'document_type' => 'pdf',
8585		- 'total_pages' => $total_pages,
8586		- 'current_page' => $page_number,
8587		- 'source_url' => $pdf_url
8588		- );
8589		-
8590		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
8591		- $page_url = esc_url($pdf_url . "#page=" . $page_number);
8592		-
8593		- // Get bot-specific embedding decision — custom-provider-aware
8594		- // (plan cbd5fd). Error code preserved.
8595		- $bot_options = $this->get_bot_options($bot_id);
8596		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
8597		-
8598		- $preflight = MxChat_Utils::embedding_preflight($options);
8599		- if (!$preflight['ok']) {
8600		- return new WP_Error('no_api_key', $preflight['reason'] . ' (bot: ' . $bot_id . ')');
8601		- }
8602		- $api_key = $preflight['api_key'];
8603		-
8604		- // Submit to database - UPDATED 2.5.6: Added content_type 'pdf'
8605		- $result = MxChat_Utils::submit_content_to_db(
8606		- $content_with_metadata,
8607		- $page_url,
8608		- $api_key,
8609		- null,
8610		- $bot_id,
8611		- 'pdf'
8612		- );
8613		-
8614		- return $result;
8615		-
8616		- } catch (Exception $e) {
8617		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
8618		- }
8619		-}
8620		-
8621		-/**
8622		- * AJAX: Get queue processing status
8623		- */
8624		-public function ajax_mxchat_get_queue_status() {
8625		- // Verify nonce and permissions
8626		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8627		-
8628		- if (!current_user_can('manage_options')) {
8629		- wp_send_json_error('Unauthorized access');
8630		- }
8631		-
8632		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8633		-
8634		- if (empty($queue_id)) {
8635		- wp_send_json_error('Missing queue ID');
8636		- }
8637		-
8638		- global $wpdb;
8639		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8640		-
8641		- // Get counts by status
8642		- $counts = $wpdb->get_results($wpdb->prepare(
8643		- "SELECT status, COUNT(*) as count
8644		- FROM $table_name
8645		- WHERE queue_id = %s
8646		- GROUP BY status",
8647		- $queue_id
8648		- ), OBJECT_K);
8649		-
8650		- $total = 0;
8651		- $completed = 0;
8652		- $failed = 0;
8653		- $processing = 0;
8654		- $pending = 0;
8655		-
8656		- foreach ($counts as $status => $data) {
8657		- $count = absint($data->count);
8658		- $total += $count;
8659		-
8660		- switch ($status) {
8661		- case 'completed':
8662		- $completed = $count;
8663		- break;
8664		- case 'failed':
8665		- $failed = $count;
8666		- break;
8667		- case 'processing':
8668		- $processing = $count;
8669		- break;
8670		- case 'pending':
8671		- $pending = $count;
8672		- break;
8673		- }
8674		- }
8675		-
8676		- // Calculate percentage
8677		- $percentage = $total > 0 ? round((($completed + $failed) / $total) * 100) : 0;
8678		-
8679		- // Get failed items details (include all failed items, not just those that exhausted retries)
8680		- $failed_items = array();
8681		- if ($failed > 0) {
8682		- $failed_items = $wpdb->get_results($wpdb->prepare(
8683		- "SELECT item_type, item_data, error_message, attempts
8684		- FROM $table_name
8685		- WHERE queue_id = %s
8686		- AND status = 'failed'
8687		- ORDER BY id DESC
8688		- LIMIT 50",
8689		- $queue_id
8690		- ));
8691		- }
8692		-
8693		- // Get queue metadata
8694		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
8695		- $queue_type = $this->mxchat_get_queue_meta($queue_id, 'queue_type');
8696		-
8697		- // Determine if queue is complete
8698		- $is_complete = ($pending === 0 && $processing === 0);
8699		-
8700		- wp_send_json_success(array(
8701		- 'queue_id' => $queue_id,
8702		- 'queue_type' => $queue_type,
8703		- 'source_url' => $source_url,
8704		- 'total' => $total,
8705		- 'completed' => $completed,
8706		- 'failed' => $failed,
8707		- 'processing' => $processing,
8708		- 'pending' => $pending,
8709		- 'percentage' => $percentage,
8710		- 'is_complete' => $is_complete,
8711		- 'failed_items' => $failed_items,
8712		- 'status' => $is_complete ? 'complete' : 'processing'
8713		- ));
8714		-}
8715		-
8716		-/**
8717		- * AJAX: Clear completed queue
8718		- */
8719		-public function ajax_mxchat_clear_queue() {
8720		- // Verify nonce and permissions
8721		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8722		-
8723		- if (!current_user_can('manage_options')) {
8724		- wp_send_json_error('Unauthorized access');
8725		- }
8726		-
8727		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8728		-
8729		- if (empty($queue_id)) {
8730		- wp_send_json_error('Missing queue ID');
8731		- }
8732		-
8733		- global $wpdb;
8734		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8735		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
8736		-
8737		- // Delete queue items
8738		- $wpdb->delete(
8739		- $table_name,
8740		- array('queue_id' => $queue_id),
8741		- array('%s')
8742		- );
8743		-
8744		- // Delete queue metadata
8745		- $wpdb->delete(
8746		- $meta_table,
8747		- array('queue_id' => $queue_id),
8748		- array('%s')
8749		- );
8750		-
8751		- wp_send_json_success(array(
8752		- 'message' => 'Queue cleared successfully'
8753		- ));
8754		-}
8755		-
8756		-/**
8757		- * AJAX: Retry failed items in queue
8758		- */
8759		-public function ajax_mxchat_retry_failed() {
8760		- // Verify nonce and permissions
8761		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8762		-
8763		- if (!current_user_can('manage_options')) {
8764		- wp_send_json_error('Unauthorized access');
8765		- }
8766		-
8767		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8768		-
8769		- if (empty($queue_id)) {
8770		- wp_send_json_error('Missing queue ID');
8771		- }
8772		-
8773		- global $wpdb;
8774		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8775		-
8776		- // Reset failed items to pending and reset attempt count
8777		- $updated = $wpdb->update(
8778		- $table_name,
8779		- array(
8780		- 'status' => 'pending',
8781		- 'attempts' => 0,
8782		- 'error_message' => null
8783		- ),
8784		- array(
8785		- 'queue_id' => $queue_id,
8786		- 'status' => 'failed'
8787		- ),
8788		- array('%s', '%d', '%s'),
8789		- array('%s', '%s')
8790		- );
8791		-
8792		- wp_send_json_success(array(
8793		- 'message' => 'Reset ' . $updated . ' failed items for retry',
8794		- 'reset_count' => $updated
8795		- ));
8796		-}
8797		-
8798		-
8799		-public function ajax_mxchat_mark_queue_complete() {
8800		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8801		-
8802		- if (!current_user_can('manage_options')) {
8803		- wp_send_json_error('Unauthorized access');
8804		- }
8805		-
8806		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8807		-
8808		- if (empty($queue_id)) {
8809		- wp_send_json_error('Missing queue ID');
8810		- }
8811		-
8812		- // Clear active queue transients
8813		- if (strpos($queue_id, 'sitemap_') === 0) {
8814		- delete_transient('mxchat_active_queue_sitemap');
8815		- } else if (strpos($queue_id, 'pdf_') === 0) {
8816		- delete_transient('mxchat_active_queue_pdf');
8817		- }
8818		-
8819		- wp_send_json_success(array('message' => 'Queue marked as complete'));
8820		-}
8821		-
8822	3553
8823	3554	// ========================================
8824	3555	// STATIC ACCESS METHODS
8825	3556	// ========================================