← All changes | admin/class-knowledge-manager.php +1587 -6057 3.2.3 → 2.2.4 View file →

		@@ -15,72 +15,49 @@
15	15
16	16	/**
17	17	* Constructor - Register hooks for content processing
18	18	*/
19		-public function __construct() {
20		- $this->options = get_option('mxchat_options', array());
21		- $this->mxchat_init_hooks();
	19	+ public function __construct() {
	20	+ $this->options = get_option('mxchat_options', array());
	21	+ $this->mxchat_init_hooks();
	22	+ }
22	23
23		- $this->mxchat_init_role_hooks();
24		-}
25		-
26		-/**
27		- * Initialize WordPress hooks for content processing
28		- *
29		- */
30		-private function mxchat_init_hooks() {
31		- // Admin post handlers for form submissions
32		- add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
33		- add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
34		- add_action('admin_post_mxchat_submit_pdf_file', array($this, 'mxchat_handle_pdf_file_submission'));
35		- add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
36		-
37		- // AJAX handlers for real-time processing and status updates
38		- add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
39		- add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status'));
40		- add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
41		- add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
42		- add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
43		- add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
44		- add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
45		- add_action('wp_ajax_mxchat_delete_chunks_by_url', array($this, 'ajax_mxchat_delete_chunks_by_url'));
46		- add_action('wp_ajax_mxchat_delete_wordpress_prompt', array($this, 'ajax_mxchat_delete_wordpress_prompt'));
47		- add_action('wp_ajax_mxchat_bulk_delete_knowledge', array($this, 'ajax_mxchat_bulk_delete_knowledge'));
48		- add_action('wp_ajax_mxchat_update_role_restriction', array($this, 'ajax_mxchat_update_role_restriction'));
49		-
50		- // Queue-based processing AJAX handlers
51		- add_action('wp_ajax_mxchat_get_next_queue_item', array($this, 'ajax_mxchat_get_next_queue_item'));
52		- add_action('wp_ajax_mxchat_process_queue_item', array($this, 'ajax_mxchat_process_queue_item'));
53		- add_action('wp_ajax_mxchat_get_queue_status', array($this, 'ajax_mxchat_get_queue_status'));
54		- add_action('wp_ajax_mxchat_clear_queue', array($this, 'ajax_mxchat_clear_queue'));
55		- add_action('wp_ajax_mxchat_retry_failed', array($this, 'ajax_mxchat_retry_failed'));
56		- add_action('wp_ajax_mxchat_get_recent_entries', array($this, 'ajax_mxchat_get_recent_entries'));
57		- add_action('wp_ajax_mxchat_detect_sitemaps', array($this, 'ajax_mxchat_detect_sitemaps'));
58		- add_action('wp_ajax_mxchat_refresh_pinecone_entries', array($this, 'ajax_mxchat_refresh_pinecone_entries'));
59		- add_action('wp_ajax_mxchat_paginate_entries', array($this, 'ajax_mxchat_paginate_entries'));
60		- add_action('wp_ajax_mxchat_get_entry_content', array($this, 'ajax_mxchat_get_entry_content'));
61		- add_action('wp_ajax_mxchat_save_entry_content', array($this, 'ajax_mxchat_save_entry_content'));
	24	+ /**
	25	+ * Initialize WordPress hooks for content processing
	26	+ */
	27	+ private function mxchat_init_hooks() {
	28	+ // Admin post handlers for form submissions
	29	+ add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
	30	+ add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
	31	+ add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
	32	+
	33	+ // AJAX handlers for real-time processing and status updates
	34	+ add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
	35	+ add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status')); // NEW
	36	+ add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
	37	+ add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
	38	+ add_action('wp_ajax_mxchat_manual_batch_process', array($this, 'ajax_manual_batch_process'));
	39	+ add_action('mxchat_delete_content', array($this, 'mxchat_delete_from_pinecone_by_url'), 10, 1);
62	40
63		- // WordPress post management hooks
64		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
65		- add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
66		- add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
67		- add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
	41	+
	42	+ // Cron handlers for background processing
	43	+ add_action('mxchat_process_sitemap_urls', array($this, 'mxchat_process_sitemap_urls_cron'), 10, 5);
	44	+ add_action('mxchat_process_pdf_pages', array($this, 'mxchat_process_pdf_pages_cron'), 10, 5);
	45	+
	46	+ // WordPress post management hooks
	47	+ add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
	48	+ add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
	49	+ add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
	50	+ add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
68	51
69		- // ACF hook - fires AFTER ACF fields are saved, ensuring ACF data is available
70		- // Priority 20 to run after ACF's own save (which runs at priority 10)
71		- add_action('acf/save_post', array($this, 'mxchat_handle_acf_save'), 20);
72		-
73		- add_action('wp_ajax_mxchat_mark_queue_complete', array($this, 'ajax_mxchat_mark_queue_complete'));
74		-
75		- // WooCommerce product hooks (if WooCommerce is active)
76		- if (class_exists('WooCommerce')) {
77		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
78		- add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
79		- add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
80		- add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
	52	+ // WooCommerce product hooks (if WooCommerce is active)
	53	+ if (class_exists('WooCommerce')) {
	54	+ add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
	55	+ add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
	56	+ add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
	57	+ }
	58	+
81	59	}
82		-}
83	60
84	61	/**
85	62	* Get current options (refreshed)
86	63	*/
		@@ -90,9 +67,315 @@
90	67	}
91	68	return $this->options;
92	69	}
93	70
	71	+
	72	+ /**
	73	+ * Handle manual batch processing via AJAX
	74	+ */
	75	+public function ajax_manual_batch_process() {
	76	+ try {
	77	+ // Verify nonce and permissions
	78	+ check_ajax_referer('mxchat_status_nonce', 'nonce');
	79	+
	80	+ if (!current_user_can('manage_options')) {
	81	+ wp_send_json_error('Unauthorized access');
	82	+ }
	83	+
	84	+ $process_type = sanitize_text_field($_POST['process_type'] ?? '');
	85	+ $url = sanitize_text_field($_POST['url'] ?? '');
	86	+
	87	+ if (empty($process_type) || empty($url)) {
	88	+ wp_send_json_error('Missing required parameters');
	89	+ }
	90	+
	91	+ $processed = 0;
	92	+
	93	+ if ($process_type === 'pdf') {
	94	+ $processed = $this->mxchat_manual_process_pdf_batch($url);
	95	+ } elseif ($process_type === 'sitemap') {
	96	+ $processed = $this->mxchat_manual_process_sitemap_batch($url);
	97	+ }
	98	+
	99	+ if ($processed > 0) {
	100	+ wp_send_json_success(array(
	101	+ 'message' => "Processed {$processed} items successfully",
	102	+ 'processed' => $processed
	103	+ ));
	104	+ } else {
	105	+ wp_send_json_error('No items were processed');
	106	+ }
	107	+
	108	+ } catch (Exception $e) {
	109	+ //error_log('Manual batch process error: ' . $e->getMessage());
	110	+ wp_send_json_error('Processing failed: ' . $e->getMessage());
	111	+ }
	112	+}
94	113
	114	+/**
	115	+ * Process a small PDF batch manually - DIRECT PROCESSING
	116	+ */
	117	+private function mxchat_manual_process_pdf_batch($pdf_url) {
	118	+ try {
	119	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	120	+ $status = get_transient($status_key);
	121	+
	122	+ if (!$status || $status['status'] !== 'processing') {
	123	+ //error_log('Manual PDF: No processing status found');
	124	+ return 0;
	125	+ }
	126	+
	127	+ //error_log('Manual PDF: Starting direct processing for ' . $pdf_url);
	128	+
	129	+ // Get current progress
	130	+ $current_page = $status['processed_pages'] ?? 0;
	131	+ $total_pages = $status['total_pages'] ?? 0;
	132	+
	133	+ if ($current_page >= $total_pages) {
	134	+ //error_log('Manual PDF: Already completed');
	135	+ return 0;
	136	+ }
	137	+
	138	+ // Try to download the PDF again for processing
	139	+ $response = wp_remote_get($pdf_url, array('timeout' => 30));
	140	+
	141	+ if (is_wp_error($response)) {
	142	+ //error_log('Manual PDF: Failed to download PDF: ' . $response->get_error_message());
	143	+ return 0;
	144	+ }
	145	+
	146	+ $pdf_content = wp_remote_retrieve_body($response);
	147	+ if (empty($pdf_content)) {
	148	+ //error_log('Manual PDF: Empty PDF content');
	149	+ return 0;
	150	+ }
	151	+
	152	+ // Save PDF temporarily
	153	+ $upload_dir = wp_upload_dir();
	154	+ $temp_pdf_path = trailingslashit($upload_dir['path']) . 'temp_manual_' . time() . '.pdf';
	155	+ file_put_contents($temp_pdf_path, $pdf_content);
	156	+
	157	+ // Process 2 pages directly
	158	+ $processed = $this->mxchat_process_pdf_pages_direct($temp_pdf_path, $pdf_url, $current_page, 5);
	159	+
	160	+ // Clean up temp file
	161	+ if (file_exists($temp_pdf_path)) {
	162	+ wp_delete_file($temp_pdf_path);
	163	+ }
	164	+
	165	+ //error_log('Manual PDF: Processed ' . $processed . ' pages');
	166	+ return $processed;
	167	+
	168	+ } catch (Exception $e) {
	169	+ //error_log('Manual PDF batch error: ' . $e->getMessage());
	170	+ return 0;
	171	+ }
	172	+}
	173	+
	174	+/**
	175	+ * Process PDF pages directly without cron
	176	+ */
	177	+private function mxchat_process_pdf_pages_direct($pdf_path, $pdf_url, $start_page, $batch_size) {
	178	+ try {
	179	+ if (!file_exists($pdf_path)) {
	180	+ //error_log('Direct PDF: File not found at ' . $pdf_path);
	181	+ return 0;
	182	+ }
	183	+
	184	+ $parser = new \Smalot\PdfParser\Parser();
	185	+ $pdf = $parser->parseFile($pdf_path);
	186	+ $pages = $pdf->getPages();
	187	+
	188	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	189	+ $status = get_transient($status_key);
	190	+
	191	+ if (!$status) {
	192	+ return 0;
	193	+ }
	194	+
	195	+ $options = get_option('mxchat_options');
	196	+ $api_key = $options['api_key'] ?? '';
	197	+
	198	+ if (empty($api_key)) {
	199	+ //error_log('Direct PDF: No API key');
	200	+ return 0;
	201	+ }
	202	+
	203	+ $processed = 0;
	204	+ $end_page = min($start_page + $batch_size, count($pages));
	205	+
	206	+ for ($i = $start_page; $i < $end_page; $i++) {
	207	+ try {
	208	+ $page_number = $i + 1;
	209	+ $text = $pages[$i]->getText();
	210	+
	211	+ if (empty($text)) {
	212	+ //error_log('Direct PDF: Empty text on page ' . $page_number);
	213	+ continue;
	214	+ }
	215	+
	216	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
	217	+ if (empty($sanitized_content)) {
	218	+ //error_log('Direct PDF: No content after sanitization on page ' . $page_number);
	219	+ continue;
	220	+ }
	221	+
	222	+ // Generate embedding
	223	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	224	+ if (is_string($embedding_vector)) {
	225	+ //error_log('Direct PDF: Embedding failed on page ' . $page_number . ': ' . $embedding_vector);
	226	+ continue;
	227	+ }
	228	+
	229	+ // Create metadata
	230	+ $metadata = array(
	231	+ 'document_type' => 'pdf',
	232	+ 'total_pages' => count($pages),
	233	+ 'current_page' => $page_number,
	234	+ 'source_url' => $pdf_url
	235	+ );
	236	+
	237	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	238	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
	239	+
	240	+ // Store in database
	241	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $api_key);
	242	+
	243	+ if (is_wp_error($db_result)) {
	244	+ //error_log('Direct PDF: DB error on page ' . $page_number . ': ' . $db_result->get_error_message());
	245	+ continue;
	246	+ }
	247	+
	248	+ $processed++;
	249	+ //error_log('Direct PDF: Successfully processed page ' . $page_number);
	250	+
	251	+ // Update status
	252	+ $status['processed_pages'] = $i + 1;
	253	+ $status['last_update'] = time();
	254	+ $status['percentage'] = round(($status['processed_pages'] / $status['total_pages']) * 100);
	255	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	256	+
	257	+ } catch (Exception $e) {
	258	+ //error_log('Direct PDF: Error processing page ' . ($i + 1) . ': ' . $e->getMessage());
	259	+ continue;
	260	+ }
	261	+ }
	262	+
	263	+ // Check if completed
	264	+ if ($status['processed_pages'] >= $status['total_pages']) {
	265	+ $status['status'] = 'complete';
	266	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	267	+ //error_log('Direct PDF: Processing completed');
	268	+ }
	269	+
	270	+ return $processed;
	271	+
	272	+ } catch (Exception $e) {
	273	+ //error_log('Direct PDF processing error: ' . $e->getMessage());
	274	+ return 0;
	275	+ }
	276	+}
	277	+
	278	+/**
	279	+ * Process a small sitemap batch manually - DIRECT PROCESSING
	280	+ */
	281	+private function mxchat_manual_process_sitemap_batch($sitemap_url) {
	282	+ try {
	283	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	284	+ $status = get_transient($status_key);
	285	+
	286	+ if (!$status || $status['status'] !== 'processing') {
	287	+ return 0;
	288	+ }
	289	+
	290	+ //error_log('Manual Sitemap: Starting direct processing for ' . $sitemap_url);
	291	+
	292	+ // Re-fetch the sitemap to get URLs
	293	+ $response = wp_remote_get($sitemap_url, array('timeout' => 30));
	294	+ if (is_wp_error($response)) {
	295	+ //error_log('Manual Sitemap: Failed to fetch sitemap');
	296	+ return 0;
	297	+ }
	298	+
	299	+ $sitemap_content = wp_remote_retrieve_body($response);
	300	+ $xml = simplexml_load_string($sitemap_content);
	301	+
	302	+ if (!$xml) {
	303	+ //error_log('Manual Sitemap: Invalid XML');
	304	+ return 0;
	305	+ }
	306	+
	307	+ $urls = array();
	308	+ foreach ($xml->url as $url_element) {
	309	+ $urls[] = (string)$url_element->loc;
	310	+ }
	311	+
	312	+ $current_processed = $status['processed_urls'] ?? 0;
	313	+ $batch_size = 5;
	314	+ $processed = 0;
	315	+
	316	+ // Process next 2 URLs
	317	+ for ($i = $current_processed; $i < min($current_processed + $batch_size, count($urls)); $i++) {
	318	+ $url = $urls[$i];
	319	+
	320	+ if ($this->mxchat_process_single_url_direct($url)) {
	321	+ $processed++;
	322	+ }
	323	+
	324	+ // Update status
	325	+ $status['processed_urls'] = $i + 1;
	326	+ $status['last_update'] = time();
	327	+ $status['percentage'] = round(($status['processed_urls'] / $status['total_urls']) * 100);
	328	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	329	+ }
	330	+
	331	+ // Check if completed
	332	+ if ($status['processed_urls'] >= $status['total_urls']) {
	333	+ $status['status'] = 'complete';
	334	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	335	+ }
	336	+
	337	+ //error_log('Manual Sitemap: Processed ' . $processed . ' URLs');
	338	+ return $processed;
	339	+
	340	+ } catch (Exception $e) {
	341	+ //error_log('Manual sitemap batch error: ' . $e->getMessage());
	342	+ return 0;
	343	+ }
	344	+}
	345	+
	346	+/**
	347	+ * Process a single URL directly
	348	+ */
	349	+private function mxchat_process_single_url_direct($url) {
	350	+ try {
	351	+ $response = wp_remote_get($url, array('timeout' => 30));
	352	+ if (is_wp_error($response)) {
	353	+ return false;
	354	+ }
	355	+
	356	+ $html = wp_remote_retrieve_body($response);
	357	+ $content = $this->mxchat_extract_main_content($html);
	358	+ $sanitized = $this->mxchat_sanitize_content_for_api($content);
	359	+
	360	+ if (empty($sanitized)) {
	361	+ return false;
	362	+ }
	363	+
	364	+ $options = get_option('mxchat_options');
	365	+ $api_key = $options['api_key'] ?? '';
	366	+
	367	+ $result = MxChat_Utils::submit_content_to_db($sanitized, $url, $api_key);
	368	+
	369	+ return !is_wp_error($result);
	370	+
	371	+ } catch (Exception $e) {
	372	+ //error_log('Single URL processing error: ' . $e->getMessage());
	373	+ return false;
	374	+ }
	375	+}
	376	+
	377	+
95	378	// ========================================
96	379	// MAIN CONTENT SUBMISSION HANDLERS
97	380	// ========================================
98	381
		@@ -108,18 +391,13 @@
108	391	wp_die(esc_html__('Nonce verification failed.', 'mxchat'));
109	392	}
110	393
111	394	// Sanitize the inputs.
112		- // Use wp_kses_post to allow safe HTML and wp_unslash to remove WordPress added slashes
113		- $article_content = wp_kses_post(wp_unslash($_POST['article_content']));
	395	+ $article_content = sanitize_textarea_field($_POST['article_content']);
114	396	$article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
115	397
116		- // Get bot_id from form submission
117		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
118		-
119		- // Get bot-specific options and API key
120		- $bot_options = $this->get_bot_options($bot_id);
121		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	398	+ // Get API key for submission
	399	+ $options = get_option('mxchat_options');
122	400	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
123	401
124	402	if (strpos($selected_model, 'voyage') === 0) {
125	403	$api_key = $options['voyage_api_key'] ?? '';
		@@ -137,10 +415,10 @@
137	415	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
138	416	exit;
139	417	}
140	418
141		- // Use centralized utility function with bot_id
142		- $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key, null, $bot_id);
	419	+ // Use centralized utility function for storage
	420	+ $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key);
143	421
144	422	if (is_wp_error($result)) {
145	423	set_transient('mxchat_admin_notice_error',
146	424	esc_html__('Error storing content: ', 'mxchat') . $result->get_error_message(),
		@@ -155,23 +433,17 @@
155	433
156	434	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
157	435	exit;
158	436	}
159		-
160	437	public function mxchat_is_pdf_url($url, $response) {
161	438	$content_type = wp_remote_retrieve_header($response, 'content-type');
162	439	$file_extension = strtolower(pathinfo($url, PATHINFO_EXTENSION));
163	440
164		- // Check Content-Disposition header for .pdf filename (Google Drive sends this)
165		- $disposition = wp_remote_retrieve_header($response, 'content-disposition');
166		- $has_pdf_disposition = ! empty($disposition) && stripos($disposition, '.pdf') !== false;
167		-
168		- return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf' || $has_pdf_disposition;
	441	+ return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf';
169	442	}
170		-
171		-
172		-public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response, $bot_id = 'default') {
	443	+public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response) {
173	444	if (!current_user_can('manage_options')) {
	445	+ //error_log(esc_html__('Unauthorized PDF processing attempt', 'mxchat'));
174	446	return false;
175	447	}
176	448
177	449	$pdf_url = esc_url_raw($pdf_url);
		@@ -177,8 +449,9 @@
177	449	$pdf_url = esc_url_raw($pdf_url);
178	450	$upload_dir = wp_upload_dir();
179	451
180	452	if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
	453	+ //error_log(sprintf(esc_html__('Upload directory error: %s', 'mxchat'), esc_html($upload_dir['error'])));
181	454	return false;
182	455	}
183	456
184	457	$pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
		@@ -185,12 +458,14 @@
185	458	$pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
186	459
187	460	$response_body = wp_remote_retrieve_body($response);
188	461	if (empty($response_body)) {
	462	+ //error_log(esc_html__('Empty PDF response body', 'mxchat'));
189	463	return false;
190	464	}
191	465
192	466	if (!wp_mkdir_p(dirname($pdf_path))) {
	467	+ //error_log(sprintf(esc_html__('Failed to create directory for PDF: %s', 'mxchat'), esc_html($pdf_path)));
193	468	return false;
194	469	}
195	470
196	471	try {
		@@ -199,694 +474,295 @@
199	474	if (!file_exists($pdf_path)) {
200	475	throw new Exception(__('Failed to save PDF file', 'mxchat'));
201	476	}
202	477
203		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
204		-
205		- if ($total_pages === false || $total_pages < 1) {
206		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
207		- }
	478	+ $parser = new \Smalot\PdfParser\Parser();
	479	+ $pdf = $parser->parseFile($pdf_path);
	480	+ $total_pages = absint(count($pdf->getPages()));
208	481
209		- // Create unique queue ID
210		- $queue_id = 'pdf_' . md5($pdf_url . time());
211		-
212		- // Create array of pages to process
213		- $pages = array();
214		- for ($i = 1; $i <= $total_pages; $i++) {
215		- $pages[] = array(
216		- 'pdf_path' => $pdf_path,
217		- 'pdf_url' => $pdf_url,
218		- 'page_number' => $i,
219		- 'total_pages' => $total_pages
220		- );
	482	+ if ($total_pages < 1) {
	483	+ throw new Exception(__('Invalid PDF: no pages found', 'mxchat'));
221	484	}
222	485
223		- // Add pages to queue
224		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
	486	+ wp_schedule_single_event(time(), 'mxchat_process_pdf_pages', array(
	487	+ 'pdf_path' => $pdf_path,
	488	+ 'pdf_url' => $pdf_url,
	489	+ 'total_pages' => $total_pages,
	490	+ 'batch_size' => absint(15),
	491	+ 'batch_pause' => absint(10)
	492	+ ));
225	493
226		- if ($queued_count === 0) {
227		- wp_delete_file($pdf_path);
228		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
229		- }
	494	+ $status_data = array(
	495	+ 'total_pages' => $total_pages,
	496	+ 'processed_pages' => 0,
	497	+ 'status' => 'processing',
	498	+ 'last_update' => time()
	499	+ );
230	500
231		- // Store queue metadata
232		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $pdf_url);
233		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
234		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
235		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
236		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
237		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	501	+ set_transient(
	502	+ sanitize_key('mxchat_pdf_status_' . md5($pdf_url)),
	503	+ array_map('sanitize_text_field', $status_data),
	504	+ DAY_IN_SECONDS
	505	+ );
238	506
239		- // Store queue ID in transient for status tracking
240		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
241		- set_transient('mxchat_last_pdf_url', $pdf_url, DAY_IN_SECONDS);
	507	+ return __('scheduled', 'mxchat');
242	508
243		- return 'queued';
244		-
245	509	} catch (Exception $e) {
	510	+ //error_log(sprintf(esc_html__('Error preparing PDF for processing: %s', 'mxchat'), esc_html($e->getMessage())));
246	511	if (file_exists($pdf_path)) {
247	512	wp_delete_file($pdf_path);
248	513	}
249		- return $e->getMessage();
	514	+ return false;
250	515	}
251	516	}
252	517
253		-/**
254		- * Handle direct PDF file upload from the knowledge base page
255		- */
256		-public function mxchat_handle_pdf_file_submission() {
257		- if (!isset($_POST['submit_pdf_file']) || !current_user_can('manage_options')) {
258		- wp_die(esc_html__('Unauthorized access', 'mxchat'));
259		- }
	518	+public function mxchat_process_pdf_pages_cron($pdf_path, $pdf_url, $total_pages, $batch_size, $batch_pause) {
	519	+ // Validate inputs
	520	+ $pdf_path = sanitize_text_field($pdf_path);
	521	+ $pdf_url = esc_url_raw($pdf_url);
	522	+ $total_pages = absint($total_pages);
	523	+ $batch_size = absint($batch_size);
	524	+ $batch_pause = absint($batch_pause);
260	525
261		- check_admin_referer('mxchat_submit_pdf_file_action', 'mxchat_submit_pdf_file_nonce');
262		-
263		- $redirect_url = admin_url('admin.php?page=mxchat-prompts');
264		-
265		- // Validate file upload
266		- if (empty($_FILES['pdf_file']) || $_FILES['pdf_file']['error'] !== UPLOAD_ERR_OK) {
267		- $error_code = isset($_FILES['pdf_file']['error']) ? $_FILES['pdf_file']['error'] : UPLOAD_ERR_NO_FILE;
268		- $error_messages = array(
269		- UPLOAD_ERR_INI_SIZE => __('The uploaded file exceeds the server upload_max_filesize limit.', 'mxchat'),
270		- UPLOAD_ERR_FORM_SIZE => __('The uploaded file exceeds the form MAX_FILE_SIZE limit.', 'mxchat'),
271		- UPLOAD_ERR_PARTIAL => __('The file was only partially uploaded.', 'mxchat'),
272		- UPLOAD_ERR_NO_FILE => __('No file was uploaded. Please select a PDF file.', 'mxchat'),
273		- UPLOAD_ERR_NO_TMP_DIR => __('Server missing temporary folder.', 'mxchat'),
274		- UPLOAD_ERR_CANT_WRITE => __('Server failed to write file to disk.', 'mxchat'),
275		- );
276		- $error_msg = isset($error_messages[$error_code]) ? $error_messages[$error_code] : __('Unknown upload error.', 'mxchat');
277		- set_transient('mxchat_admin_notice_error', $error_msg, 30);
278		- wp_safe_redirect(esc_url($redirect_url));
279		- exit;
280		- }
281		-
282		- $file = $_FILES['pdf_file'];
283		-
284		- // Validate MIME type
285		- $finfo = finfo_open(FILEINFO_MIME_TYPE);
286		- $mime_type = finfo_file($finfo, $file['tmp_name']);
287		- finfo_close($finfo);
288		-
289		- if ($mime_type !== 'application/pdf') {
290		- set_transient('mxchat_admin_notice_error',
291		- esc_html__('Invalid file type. Only PDF files are accepted.', 'mxchat'),
292		- 30
293		- );
294		- wp_safe_redirect(esc_url($redirect_url));
295		- exit;
296		- }
297		-
298		- // Validate extension
299		- $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
300		- if ($ext !== 'pdf') {
301		- set_transient('mxchat_admin_notice_error',
302		- esc_html__('Invalid file extension. Only .pdf files are accepted.', 'mxchat'),
303		- 30
304		- );
305		- wp_safe_redirect(esc_url($redirect_url));
306		- exit;
307		- }
308		-
309		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
310		- $original_filename = sanitize_file_name($file['name']);
311		-
312		- $upload_dir = wp_upload_dir();
313		- if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
314		- set_transient('mxchat_admin_notice_error',
315		- esc_html__('WordPress upload directory is not writable.', 'mxchat'),
316		- 30
317		- );
318		- wp_safe_redirect(esc_url($redirect_url));
319		- exit;
320		- }
321		-
322		- $pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
323		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
324		-
325		- if (!wp_mkdir_p(dirname($pdf_path))) {
326		- set_transient('mxchat_admin_notice_error',
327		- esc_html__('Failed to create upload directory.', 'mxchat'),
328		- 30
329		- );
330		- wp_safe_redirect(esc_url($redirect_url));
331		- exit;
332		- }
333		-
334		- // Move uploaded file
335		- if (!move_uploaded_file($file['tmp_name'], $pdf_path)) {
336		- set_transient('mxchat_admin_notice_error',
337		- esc_html__('Failed to save uploaded PDF file.', 'mxchat'),
338		- 30
339		- );
340		- wp_safe_redirect(esc_url($redirect_url));
341		- exit;
342		- }
343		-
344	526	try {
345		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
346		-
347		- if ($total_pages === false || $total_pages < 1) {
348		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
	527	+ if (!file_exists($pdf_path)) {
	528	+ throw new Exception(sprintf('PDF file not found at path: %s', esc_html($pdf_path)));
349	529	}
350	530
351		- // Use original filename as the source identifier
352		- $source_label = 'upload://' . $original_filename;
353		-
354		- $queue_id = 'pdf_' . md5($source_label . time());
355		-
356		- $pages = array();
357		- for ($i = 1; $i <= $total_pages; $i++) {
358		- $pages[] = array(
359		- 'pdf_path' => $pdf_path,
360		- 'pdf_url' => $source_label,
361		- 'page_number' => $i,
362		- 'total_pages' => $total_pages,
363		- );
364		- }
365		-
366		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
367		-
368		- if ($queued_count === 0) {
369		- wp_delete_file($pdf_path);
370		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
371		- }
372		-
373		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $source_label);
374		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
375		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
376		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
377		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
378		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
379		-
380		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
381		- set_transient('mxchat_last_pdf_url', $source_label, DAY_IN_SECONDS);
382		-
383		- set_transient('mxchat_admin_notice_success',
384		- sprintf(
385		- esc_html__('PDF "%s" (%d pages) queued for processing. Processing will start automatically.', 'mxchat'),
386		- esc_html($original_filename),
387		- $total_pages
388		- ),
389		- 30
390		- );
391		-
392		- } catch (Exception $e) {
393		- if (file_exists($pdf_path)) {
394		- wp_delete_file($pdf_path);
395		- }
396		- set_transient('mxchat_admin_notice_error',
397		- esc_html__('Failed to process uploaded PDF: ', 'mxchat') . esc_html($e->getMessage()),
398		- 30
399		- );
400		- }
401		-
402		- wp_safe_redirect(esc_url($redirect_url));
403		- exit;
404		-}
405		-
406		-/**
407		- * Validate PDF and count pages with multiple parser attempts
408		- */
409		-private function mxchat_validate_and_count_pdf_pages($pdf_path) {
410		- // Method 1: Try with Smalot PDF Parser (your current method)
411		- try {
412		- mxchat_load_pdf_parser();
413	531	$parser = new \Smalot\PdfParser\Parser();
414	532	$pdf = $parser->parseFile($pdf_path);
415	533	$pages = $pdf->getPages();
416		- $page_count = count($pages);
417		-
418		- if ($page_count > 0) {
419		- //error_log('PDF parsed successfully with Smalot parser: ' . $page_count . ' pages');
420		- return $page_count;
421		- }
422		- } catch (Exception $e) {
423		- //error_log('Smalot PDF parser failed: ' . $e->getMessage());
424		- }
425	534
426		- // Method 2: Try with pdfinfo command (if available)
427		- if (function_exists('shell_exec') && !$this->mxchat_is_shell_disabled()) {
428		- try {
429		- $command = 'pdfinfo ' . escapeshellarg($pdf_path) . ' 2>&1';
430		- $output = shell_exec($command);
431		-
432		- if ($output && preg_match('/Pages:\s*(\d+)/', $output, $matches)) {
433		- $page_count = intval($matches[1]);
434		- if ($page_count > 0) {
435		- //error_log('PDF parsed successfully with pdfinfo: ' . $page_count . ' pages');
436		- return $page_count;
437		- }
438		- }
439		- } catch (Exception $e) {
440		- //error_log('pdfinfo command failed: ' . $e->getMessage());
	535	+ // Get current progress
	536	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	537	+ $status = get_transient($status_key);
	538	+
	539	+ if (!$status || !is_array($status)) {
	540	+ throw new Exception('Invalid status data retrieved from transient');
441	541	}
442		- }
443	542
444		- // Method 3: Try to repair PDF and parse again
445		- try {
446		- $repaired_path = $this->mxchat_attempt_pdf_repair($pdf_path);
447		- if ($repaired_path && $repaired_path !== $pdf_path) {
448		- mxchat_load_pdf_parser();
449		- $parser = new \Smalot\PdfParser\Parser();
450		- $pdf = $parser->parseFile($repaired_path);
451		- $pages = $pdf->getPages();
452		- $page_count = count($pages);
453		-
454		- if ($page_count > 0) {
455		- // Replace original with repaired version
456		- copy($repaired_path, $pdf_path);
457		- unlink($repaired_path);
458		- //error_log('PDF repaired and parsed successfully: ' . $page_count . ' pages');
459		- return $page_count;
460		- }
461		-
462		- // Clean up repaired file if it didn't work
463		- unlink($repaired_path);
	543	+ // Initialize failed pages list if it doesn't exist
	544	+ if (!isset($status['failed_pages_list']) || !is_array($status['failed_pages_list'])) {
	545	+ $status['failed_pages_list'] = [];
464	546	}
465		- } catch (Exception $e) {
466		- //error_log('PDF repair attempt failed: ' . $e->getMessage());
467		- }
468	547
469		- // Method 4: Manual PDF structure analysis (basic page count)
470		- try {
471		- $page_count = $this->mxchat_manual_pdf_page_count($pdf_path);
472		- if ($page_count > 0) {
473		- //error_log('PDF page count determined manually: ' . $page_count . ' pages');
474		- return $page_count;
	548	+ $start_page = absint($status['processed_pages']);
	549	+ $end_page = min($start_page + $batch_size, $total_pages);
	550	+ $options = get_option('mxchat_options');
	551	+
	552	+ if (empty($options['api_key'])) {
	553	+ throw new Exception('API key is missing or invalid');
475	554	}
476		- } catch (Exception $e) {
477		- //error_log('Manual PDF analysis failed: ' . $e->getMessage());
478		- }
479	555
480		- //error_log('All PDF parsing methods failed for: ' . $pdf_path);
481		- return false;
482		-}
	556	+ $successful_pages = 0;
	557	+ $failed_pages = 0;
483	558
484		-/**
485		- * Check if shell_exec is disabled
486		- */
487		-private function mxchat_is_shell_disabled() {
488		- $disabled = explode(',', ini_get('disable_functions'));
489		- return in_array('shell_exec', $disabled);
490		-}
	559	+ for ($i = $start_page; $i < $end_page; $i++) {
	560	+ $page_number = $i + 1;
	561	+ $max_retries = 3;
	562	+ $retry_count = 0;
	563	+ $page_processed = false;
	564	+ $last_error = '';
491	565
492		-/**
493		- * Attempt to repair PDF using basic methods
494		- */
495		-private function mxchat_attempt_pdf_repair($pdf_path) {
496		- try {
497		- $content = file_get_contents($pdf_path);
498		- if (!$content) {
499		- return false;
500		- }
	566	+ while (!$page_processed && $retry_count < $max_retries) {
	567	+ try {
	568	+ $text = $pages[$i]->getText();
	569	+
	570	+ if (empty($text)) {
	571	+ throw new Exception("Empty text on page {$page_number}");
	572	+ }
	573	+
	574	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
501	575
502		- // Check if PDF starts with proper header
503		- if (substr($content, 0, 4) !== '%PDF') {
504		- // Try to find PDF header in the content
505		- $header_pos = strpos($content, '%PDF');
506		- if ($header_pos !== false && $header_pos < 1024) {
507		- // Remove junk before PDF header
508		- $content = substr($content, $header_pos);
509		- $repaired_path = $pdf_path . '.repaired';
510		- file_put_contents($repaired_path, $content);
511		- return $repaired_path;
512		- }
513		- }
	576	+ if (empty($sanitized_content)) {
	577	+ throw new Exception("No valid content after sanitization on page {$page_number}");
	578	+ }
514	579
515		- // Check for EOF marker
516		- $content = rtrim($content);
517		- if (!preg_match('/%%EOF\s*$/', $content)) {
518		- // Add EOF marker if missing
519		- $content .= "\n%%EOF";
520		- $repaired_path = $pdf_path . '.repaired';
521		- file_put_contents($repaired_path, $content);
522		- return $repaired_path;
523		- }
	580	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	581	+
	582	+ if (is_string($embedding_vector)) {
	583	+ throw new Exception("Embedding generation failed: " . $embedding_vector);
	584	+ }
	585	+
	586	+ if (!is_array($embedding_vector)) {
	587	+ throw new Exception("Embedding generation returned unexpected result type: " . gettype($embedding_vector));
	588	+ }
	589	+
	590	+ $metadata = array(
	591	+ 'document_type' => 'pdf',
	592	+ 'total_pages' => $total_pages,
	593	+ 'current_page' => $page_number,
	594	+ 'prev_page' => $i > 0 ? $i : null,
	595	+ 'next_page' => $i < ($total_pages - 1) ? $i + 2 : null,
	596	+ 'source_url' => $pdf_url
	597	+ );
524	598
525		- } catch (Exception $e) {
526		- //error_log('PDF repair error: ' . $e->getMessage());
527		- }
	599	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	600	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
528	601
529		- return false;
530		-}
	602	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $options['api_key']);
	603	+
	604	+ if (is_wp_error($db_result)) {
	605	+ throw new Exception("Database submission failed: " . $db_result->get_error_message());
	606	+ }
531	607
532		-/**
533		- * Manual PDF page counting by analyzing PDF structure
534		- */
535		-private function mxchat_manual_pdf_page_count($pdf_path) {
536		- try {
537		- $content = file_get_contents($pdf_path);
538		- if (!$content) {
539		- return 0;
540		- }
	608	+ // Success!
	609	+ $page_processed = true;
	610	+ $successful_pages++;
	611	+
	612	+ } catch (Exception $e) {
	613	+ $retry_count++;
	614	+ $last_error = $e->getMessage();
	615	+
	616	+ //error_log("PDF page {$page_number} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	617	+
	618	+ if ($retry_count < $max_retries) {
	619	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	620	+ sleep(pow(2, $retry_count - 1));
	621	+ }
	622	+ }
	623	+ }
541	624
542		- // Method 1: Count /Type /Page objects
543		- $page_count = preg_match_all('/\/Type\s*\/Page[^s]/', $content);
544		- if ($page_count > 0) {
545		- return $page_count;
546		- }
	625	+ // If page still not processed after all retries, mark as failed
	626	+ if (!$page_processed) {
	627	+ $failed_pages++;
	628	+ $status['failed_pages_list'][] = [
	629	+ 'page' => $page_number,
	630	+ 'error' => $last_error,
	631	+ 'time' => time(),
	632	+ 'retries' => $max_retries
	633	+ ];
	634	+
	635	+ // Limit failed pages list to prevent memory issues
	636	+ if (count($status['failed_pages_list']) > 50) {
	637	+ $status['failed_pages_list'] = array_slice($status['failed_pages_list'], -50);
	638	+ }
	639	+
	640	+ //error_log("PDF page {$page_number} permanently failed after {$max_retries} attempts: " . $last_error);
	641	+ }
547	642
548		- // Method 2: Look for /Count in pages object
549		- if (preg_match('/\/Type\s*\/Pages.*?\/Count\s+(\d+)/', $content, $matches)) {
550		- return intval($matches[1]);
	643	+ // Update progress
	644	+ $status['processed_pages'] = absint($page_number);
	645	+ $status['last_update'] = time();
	646	+ $status['failed_pages'] = absint($status['failed_pages'] ?? 0) + ($page_processed ? 0 : 1);
	647	+
	648	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
551	649	}
552	650
553		- // Method 3: Count page references
554		- $page_count = preg_match_all('/\d+\s+0\s+obj\s*<<[^>]*\/Type\s*\/Page/', $content);
555		- if ($page_count > 0) {
556		- return $page_count;
557		- }
558		-
559		- } catch (Exception $e) {
560		- //error_log('Manual PDF analysis error: ' . $e->getMessage());
561		- }
562		-
563		- return 0;
564		-}
565		-
566		-
567		-public function mxchat_save_inline_prompt() {
568		- // DEBUG: Log what we're receiving
569		- //error_log('=== MXCHAT DEBUG ===');
570		- //error_log('POST data: ' . print_r($_POST, true));
571		- //error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
572		-
573		- // Check for nonce security
574		- check_ajax_referer('mxchat_save_inline_nonce', '_ajax_nonce');
575		-
576		- // If we get here, nonce passed
577		- //error_log('Nonce verification PASSED');
578		-
579		- // Verify permissions
580		- if (!current_user_can('manage_options')) {
581		- wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
582		- return;
583		- }
584		-
585		- global $wpdb;
586		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
587		-
588		- // Validate and sanitize input data
589		- $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
590		- $article_content = isset($_POST['article_content']) ? wp_kses_post(wp_unslash($_POST['article_content'])) : '';
591		- $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
592		-
593		- if ($prompt_id > 0 && !empty($article_content)) {
594		- // Re-generate the embedding vector for the updated content
595		- $embedding_vector = $this->mxchat_generate_embedding($article_content);
596		- if (is_array($embedding_vector)) {
597		- // Serialize the embedding vector before storing it
598		- $embedding_vector_serialized = serialize($embedding_vector);
599		- // Update the prompt in the database
600		- $updated = $wpdb->update(
601		- $table_name,
602		- array(
603		- 'article_content' => $article_content,
604		- 'embedding_vector' => $embedding_vector_serialized,
605		- 'source_url' => $article_url,
606		- 'embedding_model' => MxChat_Utils::get_active_embedding_model(),
607		- ),
608		- array('id' => $prompt_id),
609		- array('%s', '%s', '%s', '%s'),
610		- array('%d')
611		- );
612		- if ($updated !== false) {
613		- wp_send_json_success();
614		- } else {
615		- MxChat_Admin::mxchat_log_debug('knowledge_error', 'Database update failed when saving knowledge entry');
616		- wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
617		- }
	651	+ // Schedule next batch if needed
	652	+ if ($end_page < $total_pages) {
	653	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_pdf_pages', array(
	654	+ 'pdf_path' => $pdf_path,
	655	+ 'pdf_url' => $pdf_url,
	656	+ 'total_pages' => $total_pages,
	657	+ 'batch_size' => $batch_size,
	658	+ 'batch_pause' => $batch_pause
	659	+ ));
618	660	} else {
619		- MxChat_Admin::mxchat_log_debug('embedding_error', 'Embedding generation failed for knowledge entry');
620		- wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
621		- }
622		- } else {
623		- wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
624		- }
625		-}
626		-
627		-
628		-/**
629		- * AJAX: Get full content for editing — reassembles chunks if needed.
630		- * Works for both WordPress DB and Pinecone entries.
631		- */
632		-public function ajax_mxchat_get_entry_content() {
633		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
634		-
635		- if ( ! current_user_can('manage_options') ) {
636		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
637		- }
638		-
639		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
640		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
641		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
642		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
643		-
644		- if ( $data_source === 'pinecone' ) {
645		- // Pinecone: fetch vectors by source_url, reassemble chunks
646		- $content = $this->get_pinecone_entry_content( $source_url, $entry_id, $bot_id );
647		- } else {
648		- // WordPress DB
649		- $content = $this->get_wordpress_entry_content( $source_url, $entry_id );
650		- }
651		-
652		- if ( is_wp_error( $content ) ) {
653		- wp_send_json_error( array( 'message' => $content->get_error_message() ) );
654		- }
655		-
656		- wp_send_json_success( $content );
657		-}
658		-
659		-/**
660		- * Get content from WordPress DB — reassembles chunks by source_url.
661		- */
662		-private function get_wordpress_entry_content( $source_url, $entry_id ) {
663		- global $wpdb;
664		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
665		-
666		- // If we have a source_url, check for chunks
667		- if ( ! empty( $source_url ) && strpos( $source_url, 'mxchat://' ) !== 0 ) {
668		- $rows = $wpdb->get_results( $wpdb->prepare(
669		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
670		- $source_url
671		- ) );
672		-
673		- if ( $rows && count( $rows ) > 1 ) {
674		- // Multiple rows = chunked. Reassemble.
675		- $chunks = array();
676		- foreach ( $rows as $row ) {
677		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
678		- $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
679		- $chunks[ $index ] = $parsed['text'];
	661	+ // Processing complete
	662	+ $status['status'] = 'complete';
	663	+ $status['processed_pages'] = $total_pages;
	664	+
	665	+ // Add completion summary
	666	+ $status['completion_summary'] = [
	667	+ 'total_pages' => $total_pages,
	668	+ 'successful_pages' => $total_pages - absint($status['failed_pages'] ?? 0),
	669	+ 'failed_pages' => absint($status['failed_pages'] ?? 0),
	670	+ 'completion_time' => current_time('mysql')
	671	+ ];
	672	+
	673	+ // Save the completed status (don't delete it - let user dismiss manually)
	674	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	675	+
	676	+ // Clean up the temporary PDF file
	677	+ if (file_exists($pdf_path)) {
	678	+ wp_delete_file($pdf_path);
680	679	}
681		- ksort( $chunks );
682		- return array(
683		- 'content' => implode( "\n\n", $chunks ),
684		- 'source_url' => $source_url,
685		- 'is_chunked' => true,
686		- 'chunk_count' => count( $chunks ),
687		- 'content_type' => $rows[0]->content_type,
688		- );
689		- } elseif ( $rows && count( $rows ) === 1 ) {
690		- $parsed = MxChat_Chunker::parse_stored_chunk( $rows[0]->article_content );
691		- return array(
692		- 'content' => $parsed['text'],
693		- 'source_url' => $source_url,
694		- 'entry_id' => $rows[0]->id,
695		- 'is_chunked' => false,
696		- 'content_type' => $rows[0]->content_type,
697		- );
	680	+
	681	+ // DON'T delete the status transients here - let user dismiss manually
698	682	}
699		- }
700	683
701		- // Fallback: fetch by ID
702		- if ( $entry_id > 0 ) {
703		- $row = $wpdb->get_row( $wpdb->prepare(
704		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
705		- $entry_id
706		- ) );
707		- if ( $row ) {
708		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
709		- return array(
710		- 'content' => $parsed['text'],
711		- 'source_url' => $row->source_url,
712		- 'entry_id' => $row->id,
713		- 'is_chunked' => false,
714		- 'content_type' => $row->content_type,
	684	+ } catch (\Exception $e) {
	685	+ //error_log(sprintf('[MXCHAT-PDF] Error processing PDF: %s', $e->getMessage()));
	686	+
	687	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	688	+ $status = get_transient($status_key);
	689	+
	690	+ if (!$status || !is_array($status)) {
	691	+ $status = array(
	692	+ 'total_pages' => $total_pages,
	693	+ 'processed_pages' => 0,
	694	+ 'status' => 'error',
	695	+ 'error' => sanitize_text_field($e->getMessage()),
	696	+ 'last_update' => time()
715	697	);
	698	+ } else {
	699	+ $status['status'] = 'error';
	700	+ $status['error'] = sanitize_text_field($e->getMessage());
	701	+ $status['last_update'] = time();
716	702	}
717		- }
718		-
719		- return new WP_Error( 'not_found', 'Entry not found.' );
720		-}
721		-
722		-/**
723		- * Get content from Pinecone — fetches vectors by source_url, reassembles chunks.
724		- */
725		-private function get_pinecone_entry_content( $source_url, $entry_id, $bot_id ) {
726		- if ( ! class_exists('MxChat_Pinecone_Manager') ) {
727		- return new WP_Error( 'pinecone_unavailable', 'Pinecone manager not available.' );
728		- }
729		-
730		- // Get Pinecone config
731		- if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
732		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
733		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
734		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
735		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
736		- } else {
737		- $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
738		- $api_key = $bot_config['api_key'] ?? '';
739		- $host = $bot_config['host'] ?? '';
740		- $namespace = $bot_config['namespace'] ?? '';
741		- }
742		-
743		- if ( empty($host) || empty($api_key) ) {
744		- return new WP_Error( 'pinecone_config', 'Pinecone not configured.' );
745		- }
746		-
747		- // List vectors with the source_url prefix
748		- $base_id = md5( $source_url );
749		- $vector_ids = array( $base_id );
750		-
751		- // Find chunk vectors
752		- $list_url = "https://{$host}/vectors/list";
753		- $list_body = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
754		- if ( ! empty($namespace) ) {
755		- $list_body['namespace'] = $namespace;
756		- }
757		-
758		- $list_resp = wp_remote_post( $list_url, array(
759		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
760		- 'body' => wp_json_encode( $list_body ),
761		- 'timeout' => 15,
762		- ) );
763		-
764		- if ( ! is_wp_error($list_resp) ) {
765		- $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
766		- if ( ! empty($list_data['vectors']) ) {
767		- foreach ( $list_data['vectors'] as $v ) {
768		- $vector_ids[] = $v['id'];
769		- }
	703	+
	704	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	705	+
	706	+ if (file_exists($pdf_path)) {
	707	+ wp_delete_file($pdf_path);
770	708	}
771	709	}
772		-
773		- // Fetch vectors with metadata
774		- $fetch_url = "https://{$host}/vectors/fetch";
775		- $fetch_body = array( 'ids' => $vector_ids );
776		- if ( ! empty($namespace) ) {
777		- $fetch_body['namespace'] = $namespace;
778		- }
779		-
780		- $fetch_resp = wp_remote_post( $fetch_url, array(
781		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
782		- 'body' => wp_json_encode( $fetch_body ),
783		- 'timeout' => 15,
784		- ) );
785		-
786		- if ( is_wp_error($fetch_resp) ) {
787		- return new WP_Error( 'pinecone_fetch', 'Failed to fetch from Pinecone.' );
788		- }
789		-
790		- $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
791		- $vectors = $fetch_data['vectors'] ?? array();
792		-
793		- if ( empty($vectors) ) {
794		- return new WP_Error( 'not_found', 'Entry not found in Pinecone.' );
795		- }
796		-
797		- // Reassemble chunks
798		- $chunks = array();
799		- $content_type = 'content';
800		- foreach ( $vectors as $vid => $vector ) {
801		- $meta = $vector['metadata'] ?? array();
802		- $text = $meta['text'] ?? '';
803		- $index = $meta['chunk_index'] ?? 0;
804		- $content_type = $meta['type'] ?? 'content';
805		- $chunks[ intval($index) ] = $text;
806		- }
807		- ksort( $chunks );
808		-
809		- return array(
810		- 'content' => implode( "\n\n", $chunks ),
811		- 'source_url' => $source_url,
812		- 'is_chunked' => count($chunks) > 1,
813		- 'chunk_count' => count($chunks),
814		- 'content_type' => $content_type,
815		- );
816	710	}
817	711
818		-/**
819		- * AJAX: Save edited content — re-chunks and re-embeds as needed.
820		- * Works for both WordPress DB and Pinecone entries.
821		- */
822		-public function ajax_mxchat_save_entry_content() {
823		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
	712	+ public function mxchat_save_inline_prompt() {
	713	+ // Check for nonce security
	714	+ check_ajax_referer('mxchat_save_inline_nonce');
824	715
825		- if ( ! current_user_can('manage_options') ) {
826		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
827		- }
	716	+ // Verify permissions
	717	+ if (!current_user_can('manage_options')) {
	718	+ wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
	719	+ return;
	720	+ }
828	721
829		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
830		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
831		- $content = isset($_POST['content']) ? wp_kses_post( wp_unslash($_POST['content']) ) : '';
832		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
833		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
834		- $content_type = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : 'content';
	722	+ global $wpdb;
	723	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
835	724
836		- if ( empty($content) ) {
837		- wp_send_json_error( array( 'message' => 'Content cannot be empty.' ) );
838		- }
	725	+ // Validate and sanitize input data
	726	+ $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
	727	+ $article_content = isset($_POST['article_content']) ? sanitize_textarea_field($_POST['article_content']) : '';
	728	+ $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
839	729
840		- // Get the embedding API key
841		- $options = get_option('mxchat_options', array());
842		- $api_key = '';
	730	+ if ($prompt_id > 0 && !empty($article_content)) {
	731	+ // Re-generate the embedding vector for the updated content
	732	+ $embedding_vector = $this->mxchat_generate_embedding($article_content);
843	733
844		- if ( $bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager') ) {
845		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
846		- $api_key = $bot_options['api_key'] ?? '';
847		- }
848		- if ( empty($api_key) ) {
849		- $api_key = $options['api_key'] ?? '';
850		- }
	734	+ if (is_array($embedding_vector)) {
	735	+ // Serialize the embedding vector before storing it
	736	+ $embedding_vector_serialized = serialize($embedding_vector);
851	737
852		- global $wpdb;
853		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
	738	+ // Update the prompt in the database
	739	+ $updated = $wpdb->update(
	740	+ $table_name,
	741	+ array(
	742	+ 'article_content' => $article_content,
	743	+ 'embedding_vector' => $embedding_vector_serialized,
	744	+ 'source_url' => $article_url,
	745	+ ),
	746	+ array('id' => $prompt_id),
	747	+ array('%s', '%s', '%s'),
	748	+ array('%d')
	749	+ );
854	750
855		- // If source_url is empty but we have an entry_id, look it up
856		- if ( empty($source_url) && $entry_id > 0 && $data_source === 'wordpress' ) {
857		- $row = $wpdb->get_row( $wpdb->prepare( "SELECT source_url FROM {$table} WHERE id = %d", $entry_id ) );
858		- if ( $row && ! empty($row->source_url) ) {
859		- $source_url = $row->source_url;
860		- }
861		- }
	751	+ if ($updated !== false) {
	752	+ wp_send_json_success();
	753	+ } else {
	754	+ wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
	755	+ }
	756	+ } else {
	757	+ wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
	758	+ }
	759	+ } else {
	760	+ wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
	761	+ }
	762	+ }
862	763
863		- // For manual entries (no source_url or mxchat:// prefix), delete the old entry by ID first
864		- // so submit_content_to_db creates a replacement instead of a duplicate
865		- // Also treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
866		- $is_legacy_manual = !empty($source_url) && strpos($source_url, 'mxchat.ai') !== false && strpos($source_url, 'mxchat://') !== 0;
867		- if ( $entry_id > 0 && (empty($source_url) || strpos($source_url, 'mxchat://') === 0 || $is_legacy_manual) ) {
868		- $wpdb->delete( $table, array( 'id' => $entry_id ), array( '%d' ) );
869		- // Clear legacy URL so submit_content_to_db generates a unique mxchat:// identifier
870		- // instead of reusing the shared URL (which would mass-delete other entries with the same URL)
871		- if ( $is_legacy_manual ) {
872		- $source_url = '';
873		- }
874		- }
875	764
876		- // Use the existing submit_content_to_db which handles chunking, Pinecone, and WP DB
877		- $vector_id = ! empty($source_url) ? md5($source_url) : md5('mxchat_manual_' . $entry_id);
878		-
879		- // submit_content_to_db already handles: delete old chunks → re-chunk → re-embed → store
880		- $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, $vector_id, $bot_id, $content_type );
881		-
882		- if ( is_wp_error($result) ) {
883		- wp_send_json_error( array( 'message' => $result->get_error_message() ) );
884		- }
885		-
886		- wp_send_json_success( array( 'message' => 'Content saved and re-embedded successfully.' ) );
887		-}
888		-
889	765	public function mxchat_get_pdf_processing_status($pdf_url) {
890	766	$pdf_url = esc_url_raw($pdf_url);
891	767	$status = get_transient(sanitize_key('mxchat_pdf_status_' . md5($pdf_url)));
892	768
		@@ -929,18 +805,24 @@
929	805	}
930	806
931	807
932	808	public function mxchat_handle_sitemap_submission() {
	809	+ // Start logging the submission process
	810	+ //error_log('[MXCHAT-URL] ===== Starting URL submission process =====');
	811	+
933	812	// Check if the form was submitted and verify permissions
934	813	if (!isset($_POST['submit_sitemap']) || !current_user_can('manage_options')) {
	814	+ //error_log('[MXCHAT-URL] Error: Unauthorized access or form not submitted properly');
935	815	wp_die(esc_html__('Unauthorized access', 'mxchat'));
936	816	}
937	817
938	818	// Verify nonce
	819	+ //error_log('[MXCHAT-URL] Verifying nonce');
939	820	check_admin_referer('mxchat_submit_sitemap_action', 'mxchat_submit_sitemap_nonce');
940	821
941	822	// Validate URL
942	823	if (!isset($_POST['sitemap_url']) || empty($_POST['sitemap_url'])) {
	824	+ //error_log('[MXCHAT-URL] Error: Empty or missing URL');
943	825	set_transient('mxchat_admin_notice_error',
944	826	esc_html__('Please provide a valid URL.', 'mxchat'),
945	827	30
946	828	);
		@@ -948,28 +830,12 @@
948	830	exit;
949	831	}
950	832
951	833	$submitted_url = esc_url_raw($_POST['sitemap_url']);
	834	+ //error_log('[MXCHAT-URL] Processing URL: ' . $submitted_url);
952	835
953		- // Convert Google Drive sharing URLs to direct download URLs
954		- if ( strpos($submitted_url, 'drive.google.com') !== false ) {
955		- $file_id = '';
956		- if ( preg_match('/[?&]id=([a-zA-Z0-9_-]+)/', $submitted_url, $m) ) {
957		- $file_id = $m[1];
958		- } elseif ( preg_match('#/file/d/([a-zA-Z0-9_-]+)#', $submitted_url, $m) ) {
959		- $file_id = $m[1];
960		- }
961		- if ( ! empty($file_id) ) {
962		- $submitted_url = 'https://drive.google.com/uc?export=download&id=' . $file_id;
963		- }
964		- }
965		-
966		- // Get bot_id from form submission
967		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
968		-
969		- // Get bot-specific options and validate API key
970		- $bot_options = $this->get_bot_options($bot_id);
971		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	836	+ // Validate API key first
	837	+ $options = get_option('mxchat_options');
972	838	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
973	839
974	840	if (strpos($selected_model, 'voyage') === 0) {
975	841	$api_key = $options['voyage_api_key'] ?? '';
		@@ -986,26 +852,21 @@
986	852	$error_message = sprintf(
987	853	esc_html__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
988	854	$provider_name
989	855	);
	856	+ //error_log('[MXCHAT-URL] Error: ' . $error_message);
990	857	set_transient('mxchat_admin_notice_error', $error_message, 30);
	858	+ //error_log('[MXCHAT-URL] Set error transient: ' . $error_message);
991	859	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
992	860	exit;
993	861	}
994	862
995		- // Fetch URL — use browser-like headers so servers with bot protection don't block us
996		- $response = wp_remote_get($submitted_url, array(
997		- 'timeout' => 30,
998		- 'sslverify' => false,
999		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
1000		- 'headers' => array(
1001		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1002		- 'Accept-Language' => 'en-US,en;q=0.9',
1003		- ),
1004		- ));
	863	+ //error_log('[MXCHAT-URL] Fetching URL content');
	864	+ $response = wp_remote_get($submitted_url, array('timeout' => 30));
1005	865
1006	866	if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
1007	867	$error_message = is_wp_error($response) ? $response->get_error_message() : 'HTTP Status: ' . wp_remote_retrieve_response_code($response);
	868	+ //error_log('[MXCHAT-URL] Error fetching URL: ' . $error_message);
1008	869	set_transient('mxchat_admin_notice_error',
1009	870	sprintf(
1010	871	esc_html__('Failed to fetch the URL: %s', 'mxchat'),
1011	872	esc_html($error_message)
		@@ -1016,11 +877,13 @@
1016	877	exit;
1017	878	}
1018	879
1019	880	$content_type = wp_remote_retrieve_header($response, 'content-type');
	881	+ //error_log('[MXCHAT-URL] Content type: ' . $content_type);
1020	882	$body_content = wp_remote_retrieve_body($response);
1021	883
1022	884	if (empty($body_content)) {
	885	+ //error_log('[MXCHAT-URL] Error: Empty response body');
1023	886	set_transient('mxchat_admin_notice_error',
1024	887	esc_html__('Empty response received from URL.', 'mxchat'),
1025	888	30
1026	889	);
		@@ -1026,21 +889,29 @@
1026	889	);
1027	890	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1028	891	exit;
1029	892	}
	893	+ //error_log('[MXCHAT-URL] Retrieved body content length: ' . strlen($body_content) . ' bytes');
1030	894
1031	895	// Handle PDF URL
1032	896	if ($this->mxchat_is_pdf_url($submitted_url, $response)) {
1033		- $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response, $bot_id);
	897	+ //error_log('[MXCHAT-URL] Detected PDF URL, handling PDF for knowledge base');
	898	+ $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response);
	899	+ //error_log('[MXCHAT-URL] PDF handling result: ' . $result);
1034	900
1035		- if ($result === 'queued') {
1036		- set_transient('mxchat_admin_notice_success',
1037		- esc_html__('PDF queued for processing. Processing will start automatically.', 'mxchat'),
	901	+ if ($result === 'scheduled') {
	902	+ set_transient(
	903	+ 'mxchat_last_pdf_url',
	904	+ sanitize_text_field($submitted_url),
	905	+ DAY_IN_SECONDS
	906	+ );
	907	+ set_transient('mxchat_admin_notice_info',
	908	+ esc_html__('PDF processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1038	909	30
1039	910	);
1040	911	} else {
1041	912	set_transient('mxchat_admin_notice_error',
1042		- esc_html__('Failed to queue PDF processing: ', 'mxchat') . esc_html($result),
	913	+ esc_html__('Failed to start PDF processing: ', 'mxchat') . esc_html($result),
1043	914	30
1044	915	);
1045	916	}
1046	917
		@@ -1049,8 +920,9 @@
1049	920	}
1050	921
1051	922	// Handle Sitemap XML
1052	923	if (strpos($content_type, 'xml') !== false || strpos($body_content, '<urlset') !== false) {
	924	+ //error_log('[MXCHAT-URL] Detected XML content, processing as sitemap');
1053	925	libxml_use_internal_errors(true);
1054	926	$xml = simplexml_load_string($body_content);
1055	927	$xml_errors = libxml_get_errors();
1056	928	libxml_clear_errors();
		@@ -1055,8 +927,15 @@
1055	927	$xml_errors = libxml_get_errors();
1056	928	libxml_clear_errors();
1057	929
1058	930	if ($xml === false || !empty($xml_errors)) {
	931	+ //error_log('[MXCHAT-URL] Error: Invalid XML format');
	932	+ if (!empty($xml_errors)) {
	933	+ foreach ($xml_errors as $error) {
	934	+ //error_log('[MXCHAT-URL] XML Error: ' . $error->message);
	935	+ }
	936	+ }
	937	+
1059	938	set_transient('mxchat_admin_notice_error',
1060	939	esc_html__('Invalid sitemap XML. Please provide a valid sitemap.', 'mxchat'),
1061	940	30
1062	941	);
		@@ -1063,18 +942,27 @@
1063	942	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1064	943	exit;
1065	944	}
1066	945
1067		- $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url, $bot_id);
	946	+ //error_log('[MXCHAT-URL] Valid XML found, handling sitemap for knowledge base');
	947	+ $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url);
	948	+ //error_log('[MXCHAT-URL] Sitemap handling result: ' . $result);
1068	949
1069		- if ($result === 'queued') {
1070		- set_transient('mxchat_admin_notice_success',
1071		- esc_html__('Sitemap queued for processing. Processing will start automatically.', 'mxchat'),
	950	+ if ($result === 'scheduled') {
	951	+ set_transient(
	952	+ 'mxchat_last_sitemap_url',
	953	+ sanitize_text_field($submitted_url),
	954	+ DAY_IN_SECONDS
	955	+ );
	956	+ set_transient('mxchat_admin_notice_info',
	957	+ esc_html__('Sitemap processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1072	958	30
1073	959	);
1074	960	} else {
	961	+ // Return to the admin page without a redirect for better error display
	962	+ // The error is already stored in the sitemap status transient
1075	963	set_transient('mxchat_admin_notice_error',
1076		- esc_html__('Failed to queue sitemap processing. Please check the status below for details.', 'mxchat'),
	964	+ esc_html__('Failed to start sitemap processing. Please check the status below for details.', 'mxchat'),
1077	965	30
1078	966	);
1079	967	}
1080	968
		@@ -1081,50 +969,127 @@
1081	969	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1082	970	exit;
1083	971	}
1084	972
1085		- // Handle Regular URL (single page)
	973	+ // Handle Regular URL
	974	+ //error_log('[MXCHAT-URL] Processing as regular webpage');
1086	975	$page_content = $this->mxchat_extract_main_content($body_content);
	976	+ //error_log('[MXCHAT-URL] Extracted content length: ' . strlen($page_content) . ' bytes');
	977	+
1087	978	$sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	979	+ //error_log('[MXCHAT-URL] Sanitized content length: ' . strlen($sanitized_content) . ' bytes');
1088	980
1089		- //error_log('[MXCHAT-URL-DEBUG] URL: ' . $submitted_url);
1090		- //error_log('[MXCHAT-URL-DEBUG] Extracted page_content length: ' . strlen($page_content));
1091		- //error_log('[MXCHAT-URL-DEBUG] Sanitized content length: ' . strlen($sanitized_content));
1092		- //error_log('[MXCHAT-URL-DEBUG] Content preview: ' . substr($sanitized_content, 0, 500));
1093		-
1094	981	if (empty($sanitized_content)) {
	982	+ //error_log('[MXCHAT-URL] Error: No valid content after sanitization');
	983	+
	984	+ // Set both transients - the error notice and the URL status
1095	985	set_transient('mxchat_admin_notice_error',
1096	986	esc_html__('No valid content found on the provided URL.', 'mxchat'),
1097	987	30
1098	988	);
	989	+
	990	+ // Set URL status transient
	991	+ set_transient('mxchat_single_url_status', [
	992	+ 'url' => $submitted_url,
	993	+ 'timestamp' => current_time('mysql'),
	994	+ 'status' => 'failed',
	995	+ 'error' => esc_html__('No valid content found on the provided URL.', 'mxchat')
	996	+ ], DAY_IN_SECONDS);
	997	+
1099	998	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1100	999	exit;
1101	1000	}
1102	1001
1103		- // For single URLs, process immediately using submit_content_to_db
1104		- // This handles chunking automatically for large content
1105		- $db_result = MxChat_Utils::submit_content_to_db(
1106		- $sanitized_content,
1107		- $submitted_url,
1108		- $api_key,
1109		- null,
1110		- $bot_id,
1111		- 'url' // content_type
1112		- );
	1002	+ //error_log('[MXCHAT-URL] Generating embedding for content');
	1003	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
1113	1004
1114		- if (is_wp_error($db_result)) {
1115		- $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1005	+ // Check if embedding_vector is a string (error message)
	1006	+ if (is_string($embedding_vector)) {
	1007	+ //error_log('[MXCHAT-URL] Error generating embedding: ' . $embedding_vector);
	1008	+ $error_message = esc_html__('Failed to generate embedding: ', 'mxchat') . esc_html($embedding_vector);
	1009	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1010	+
	1011	+ // Set both transients
1116	1012	set_transient('mxchat_admin_notice_error', $error_message, 30);
1117		- } else {
	1013	+
	1014	+ // Set URL status transient
	1015	+ set_transient('mxchat_single_url_status', [
	1016	+ 'url' => $submitted_url,
	1017	+ 'timestamp' => current_time('mysql'),
	1018	+ 'status' => 'failed',
	1019	+ 'error' => $error_message
	1020	+ ], DAY_IN_SECONDS);
	1021	+
	1022	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1023	+ exit;
	1024	+ }
	1025	+
	1026	+ if (is_array($embedding_vector)) {
	1027	+ //error_log('[MXCHAT-URL] Successfully generated embedding with ' . count($embedding_vector) . ' dimensions');
	1028	+
	1029	+ $db_result = MxChat_Utils::submit_content_to_db(
	1030	+ $sanitized_content,
	1031	+ $submitted_url,
	1032	+ $api_key
	1033	+ );
	1034	+
	1035	+ if (is_wp_error($db_result)) {
	1036	+ //error_log('[MXCHAT-URL] Error: Failed to store content in database: ' . $db_result->get_error_message());
	1037	+ $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1038	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1039	+
	1040	+ // Set both transients
	1041	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1042	+
	1043	+ // Set URL status transient
	1044	+ set_transient('mxchat_single_url_status', [
	1045	+ 'url' => $submitted_url,
	1046	+ 'timestamp' => current_time('mysql'),
	1047	+ 'status' => 'failed',
	1048	+ 'error' => $error_message
	1049	+ ], DAY_IN_SECONDS);
	1050	+
	1051	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1052	+ exit;
	1053	+ }
	1054	+
	1055	+ //error_log('[MXCHAT-URL] Successfully stored content in database');
1118	1056	$success_message = esc_html__('URL content successfully submitted!', 'mxchat');
	1057	+ //error_log('[MXCHAT-URL] Setting success transient: ' . $success_message);
	1058	+
	1059	+ // Set both transients
1119	1060	set_transient('mxchat_admin_notice_success', $success_message, 30);
	1061	+
	1062	+ // Set URL status transient with success
	1063	+ set_transient('mxchat_single_url_status', [
	1064	+ 'url' => $submitted_url,
	1065	+ 'timestamp' => current_time('mysql'),
	1066	+ 'status' => 'complete',
	1067	+ 'content_length' => strlen($sanitized_content),
	1068	+ 'embedding_dimensions' => count($embedding_vector)
	1069	+ ], DAY_IN_SECONDS);
	1070	+
	1071	+ } else {
	1072	+ //error_log('[MXCHAT-URL] Error: Failed to generate embedding. Unexpected result type: ' . gettype($embedding_vector));
	1073	+ $error_message = esc_html__('Failed to generate embedding: Unexpected result type. Please check your API key and try again.', 'mxchat');
	1074	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1075	+
	1076	+ // Set both transients
	1077	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1078	+
	1079	+ // Set URL status transient
	1080	+ set_transient('mxchat_single_url_status', [
	1081	+ 'url' => $submitted_url,
	1082	+ 'timestamp' => current_time('mysql'),
	1083	+ 'status' => 'failed',
	1084	+ 'error' => $error_message
	1085	+ ], DAY_IN_SECONDS);
1120	1086	}
1121	1087
	1088	+ //error_log('[MXCHAT-URL] ===== Completed URL submission process =====');
1122	1089	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1123	1090	exit;
1124	1091	}
1125		-
1126		-
1127	1092	public function mxchat_get_single_url_status() {
1128	1093	$status = get_transient('mxchat_single_url_status');
1129	1094	if (!$status) {
1130	1095	return null;
		@@ -1136,11 +1101,13 @@
1136	1101	}
1137	1102
1138	1103	return $status;
1139	1104	}
1140		-
1141		-public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url, $bot_id = 'default') {
	1105	+public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url) {
	1106	+ // Clear any single URL status when starting sitemap processing
	1107	+ delete_transient('mxchat_single_url_status');
1142	1108	if (!current_user_can('manage_options')) {
	1109	+ //error_log(esc_html__('Unauthorized sitemap processing attempt', 'mxchat'));
1143	1110	return false;
1144	1111	}
1145	1112
1146	1113	try {
		@@ -1149,96 +1116,331 @@
1149	1116	if (!$xml || !is_object($xml)) {
1150	1117	throw new Exception(__('Invalid XML object provided', 'mxchat'));
1151	1118	}
1152	1119
1153		- // Get bot-specific embedding API for validation
1154		- $bot_options = $this->get_bot_options($bot_id);
1155		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	1120	+ // Add embedding validation before processing
	1121	+ // Test embedding with a small sample text to verify API key is working
	1122	+ $test_result = $this->mxchat_generate_embedding("This is a test to verify the embedding API key is working.");
1156	1123
1157		- // Test the embedding API before processing
1158		- $test_phrase = "Test embedding generation for MxChat";
1159		- $test_result = $this->mxchat_generate_embedding($test_phrase, $bot_id);
1160		-
	1124	+ // Check if test_result is a string (error message) rather than an array (valid embedding)
1161	1125	if (is_string($test_result)) {
	1126	+ //error_log('[MXCHAT-URL] Embedding API validation failed: ' . $test_result);
	1127	+
	1128	+ // Store the error in the status transient so it can be displayed later
	1129	+ $status_data = array(
	1130	+ 'total_urls' => 0,
	1131	+ 'processed_urls' => 0,
	1132	+ 'status' => 'error',
	1133	+ 'error' => __('Embedding API validation failed: ', 'mxchat') . $test_result,
	1134	+ 'last_update' => time()
	1135	+ );
	1136	+
	1137	+ set_transient(
	1138	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1139	+ array_map('sanitize_text_field', $status_data),
	1140	+ DAY_IN_SECONDS
	1141	+ );
	1142	+
1162	1143	throw new Exception(__('Embedding API validation failed: ', 'mxchat') . $test_result);
1163	1144	}
1164	1145
	1146	+ // Make sure it's an array (valid embedding)
1165	1147	if (!is_array($test_result)) {
	1148	+ //error_log('[MXCHAT-URL] Embedding API returned unexpected result type: ' . gettype($test_result));
1166	1149	throw new Exception(__('Embedding API returned unexpected result type. Please check your configuration.', 'mxchat'));
1167	1150	}
1168	1151
1169		- // Extract URLs from sitemap
1170		- $urls = array();
	1152	+ $urls = [];
1171	1153	foreach ($xml->url as $url_element) {
1172	1154	$url = esc_url_raw((string)$url_element->loc);
1173	1155	if ($url) {
1174		- $urls[] = array('url' => $url);
	1156	+ $urls[] = $url;
1175	1157	}
1176	1158	}
1177	1159
1178		- $total_urls = count($urls);
	1160	+ $total_urls = absint(count($urls));
1179	1161
1180	1162	if ($total_urls < 1) {
1181	1163	throw new Exception(__('No valid URLs found in sitemap', 'mxchat'));
1182	1164	}
1183	1165
1184		- // Create unique queue ID
1185		- $queue_id = 'sitemap_' . md5($sitemap_url . time());
	1166	+ wp_schedule_single_event(time(), 'mxchat_process_sitemap_urls', array(
	1167	+ 'urls' => $urls,
	1168	+ 'sitemap_url' => $sitemap_url,
	1169	+ 'total_urls' => $total_urls,
	1170	+ 'batch_size' => absint(10),
	1171	+ 'batch_pause' => absint(5)
	1172	+ ));
1186	1173
1187		- // Add URLs to queue
1188		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'url', $urls, $bot_id);
	1174	+ $status_data = array(
	1175	+ 'total_urls' => $total_urls,
	1176	+ 'processed_urls' => 0,
	1177	+ 'status' => 'processing',
	1178	+ 'last_update' => time()
	1179	+ );
1189	1180
1190		- if ($queued_count === 0) {
1191		- throw new Exception(__('Failed to add URLs to processing queue', 'mxchat'));
1192		- }
	1181	+ set_transient(
	1182	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1183	+ array_map('sanitize_text_field', $status_data),
	1184	+ DAY_IN_SECONDS
	1185	+ );
1193	1186
1194		- // Store queue metadata
1195		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $sitemap_url);
1196		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'sitemap');
1197		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_urls);
1198		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
1199		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	1187	+ return __('scheduled', 'mxchat');
1200	1188
1201		- // Store queue ID in transient for status tracking
1202		- set_transient('mxchat_active_queue_sitemap', $queue_id, DAY_IN_SECONDS);
1203		- set_transient('mxchat_last_sitemap_url', $sitemap_url, DAY_IN_SECONDS);
1204		-
1205		- return 'queued';
1206		-
1207		- } catch (Exception $e) {
	1189	+ } catch (\Exception $e) {
1208	1190	$error_message = $e->getMessage();
1209	1191	//error_log(sprintf(esc_html__('Error preparing sitemap for processing: %s', 'mxchat'), esc_html($error_message)));
1210	1192
	1193	+ // Store the sitemap URL and error in transients so they can be displayed
	1194	+ set_transient(
	1195	+ 'mxchat_last_sitemap_url',
	1196	+ sanitize_text_field($sitemap_url),
	1197	+ DAY_IN_SECONDS
	1198	+ );
	1199	+
	1200	+ $status_data = array(
	1201	+ 'total_urls' => 0,
	1202	+ 'processed_urls' => 0,
	1203	+ 'status' => 'error',
	1204	+ 'error' => $error_message,
	1205	+ 'last_update' => time()
	1206	+ );
	1207	+
	1208	+ set_transient(
	1209	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1210	+ array_map('sanitize_text_field', $status_data),
	1211	+ DAY_IN_SECONDS
	1212	+ );
	1213	+
1211	1214	return $error_message;
1212	1215	}
1213		-
1214	1216	}
1215	1217
1216		-/**
1217		- * Remove shortcode tags but preserve the content inside them
1218		- * Example: [vc_column]Hello World[/vc_column] becomes "Hello World"
1219		- *
1220		- * @param string $content The content containing shortcodes
1221		- * @return string Content with shortcode tags removed but inner content preserved
1222		- */
1223		-private function strip_shortcode_tags_preserve_content($content) {
1224		- // Single-pass regex removes all shortcode brackets: [tag], [tag attr="val"], [/tag], [tag /]
1225		- // Content between tags is inherently preserved since only brackets are targeted
1226		- $result = preg_replace('/\[\/?\w[\w-]*[^\]]*\]/', '', $content);
1227		- return ($result !== null) ? $result : $content;
	1218	+public function mxchat_process_sitemap_urls_cron($urls, $sitemap_url, $total_urls, $batch_size, $batch_pause) {
	1219	+ // Validate inputs
	1220	+ $sitemap_url = esc_url_raw($sitemap_url);
	1221	+ $total_urls = absint($total_urls);
	1222	+ $batch_size = absint($batch_size);
	1223	+ $batch_pause = absint($batch_pause);
	1224	+
	1225	+ if (!is_array($urls) || empty($urls)) {
	1226	+ return;
	1227	+ }
	1228	+
	1229	+ try {
	1230	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	1231	+ $status = get_transient($status_key);
	1232	+
	1233	+ if (!$status || !is_array($status)) {
	1234	+ throw new Exception('Invalid status data retrieved from transient');
	1235	+ }
	1236	+
	1237	+ // Initialize failed_urls array if it doesn't exist
	1238	+ if (!isset($status['failed_urls_list']) || !is_array($status['failed_urls_list'])) {
	1239	+ $status['failed_urls_list'] = [];
	1240	+ }
	1241	+
	1242	+ $start_url = absint($status['processed_urls']);
	1243	+ $end_url = min($start_url + $batch_size, $total_urls);
	1244	+
	1245	+ // Track batch statistics
	1246	+ $batch_stats = [
	1247	+ 'processed' => 0,
	1248	+ 'failed' => 0,
	1249	+ 'last_error' => '',
	1250	+ 'embedding_errors' => 0
	1251	+ ];
	1252	+
	1253	+ // Check embedding configuration with first URL (only on first batch)
	1254	+ if ($start_url === 0) {
	1255	+ $test_url = esc_url_raw($urls[0]);
	1256	+ $test_response = wp_remote_get($test_url);
	1257	+
	1258	+ if (!is_wp_error($test_response) && wp_remote_retrieve_response_code($test_response) === 200) {
	1259	+ $test_html = wp_remote_retrieve_body($test_response);
	1260	+ $test_content = $this->mxchat_extract_main_content($test_html);
	1261	+ $test_sanitized = $this->mxchat_sanitize_content_for_api($test_content);
	1262	+
	1263	+ if (!empty($test_sanitized)) {
	1264	+ $test_embedding = $this->mxchat_generate_embedding($test_sanitized);
	1265	+
	1266	+ if (is_string($test_embedding)) {
	1267	+ throw new Exception('Embedding generation failed: ' . $test_embedding);
	1268	+ }
	1269	+
	1270	+ if (!is_array($test_embedding)) {
	1271	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($test_embedding));
	1272	+ }
	1273	+ }
	1274	+ }
	1275	+ }
	1276	+
	1277	+ for ($i = $start_url; $i < $end_url; $i++) {
	1278	+ $page_url = esc_url_raw($urls[$i]);
	1279	+ $max_retries = 3;
	1280	+ $retry_count = 0;
	1281	+ $url_processed = false;
	1282	+ $last_error = '';
	1283	+
	1284	+ while (!$url_processed && $retry_count < $max_retries) {
	1285	+ try {
	1286	+ // Attempt to fetch the URL
	1287	+ $page_response = wp_remote_get($page_url, array('timeout' => 30));
	1288	+
	1289	+ if (is_wp_error($page_response)) {
	1290	+ throw new Exception('HTTP request failed: ' . $page_response->get_error_message());
	1291	+ }
	1292	+
	1293	+ $response_code = wp_remote_retrieve_response_code($page_response);
	1294	+ if ($response_code !== 200) {
	1295	+ throw new Exception('HTTP Status: ' . $response_code);
	1296	+ }
	1297	+
	1298	+ $page_html = wp_remote_retrieve_body($page_response);
	1299	+
	1300	+ if (empty($page_html)) {
	1301	+ throw new Exception('Empty response body');
	1302	+ }
	1303	+
	1304	+ $page_content = $this->mxchat_extract_main_content($page_html);
	1305	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	1306	+
	1307	+ if (empty($sanitized_content)) {
	1308	+ throw new Exception('No valid content found after processing');
	1309	+ }
	1310	+
	1311	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	1312	+
	1313	+ if (is_string($embedding_vector)) {
	1314	+ throw new Exception('Embedding generation failed: ' . $embedding_vector);
	1315	+ }
	1316	+
	1317	+ if (!is_array($embedding_vector)) {
	1318	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($embedding_vector));
	1319	+ }
	1320	+
	1321	+ // Submit to database
	1322	+ $options = get_option('mxchat_options');
	1323	+ $submission_result = MxChat_Utils::submit_content_to_db($sanitized_content, $page_url, $options['api_key']);
	1324	+
	1325	+ if (is_wp_error($submission_result)) {
	1326	+ throw new Exception('Database submission failed: ' . $submission_result->get_error_message());
	1327	+ }
	1328	+
	1329	+ // Success!
	1330	+ $url_processed = true;
	1331	+ $batch_stats['processed']++;
	1332	+
	1333	+ } catch (Exception $e) {
	1334	+ $retry_count++;
	1335	+ $last_error = $e->getMessage();
	1336	+
	1337	+ //error_log("URL {$page_url} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	1338	+
	1339	+ // Track embedding errors specifically
	1340	+ if (strpos($last_error, 'Embedding') !== false) {
	1341	+ $batch_stats['embedding_errors']++;
	1342	+ }
	1343	+
	1344	+ if ($retry_count < $max_retries) {
	1345	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	1346	+ sleep(pow(2, $retry_count - 1));
	1347	+ }
	1348	+ }
	1349	+ }
	1350	+
	1351	+ // If URL still not processed after all retries, mark as failed
	1352	+ if (!$url_processed) {
	1353	+ $batch_stats['failed']++;
	1354	+ $batch_stats['last_error'] = $last_error;
	1355	+
	1356	+ // Add to failed URLs list
	1357	+ $status['failed_urls_list'][] = [
	1358	+ 'url' => $page_url,
	1359	+ 'error' => $last_error,
	1360	+ 'time' => time(),
	1361	+ 'retries' => $max_retries
	1362	+ ];
	1363	+
	1364	+ // Limit the number of failed URLs we store to prevent transient size issues
	1365	+ if (count($status['failed_urls_list']) > 100) {
	1366	+ $status['failed_urls_list'] = array_slice($status['failed_urls_list'], -100);
	1367	+ }
	1368	+ }
	1369	+
	1370	+ // Update progress
	1371	+ $status['processed_urls'] = absint($i + 1);
	1372	+ $status['last_update'] = time();
	1373	+ $status['failed_urls'] = absint($status['failed_urls'] ?? 0) + ($url_processed ? 0 : 1);
	1374	+ $status['last_error'] = $batch_stats['last_error'];
	1375	+
	1376	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1377	+
	1378	+ // If we have too many consecutive embedding errors, stop processing
	1379	+ if ($batch_stats['embedding_errors'] >= 10) {
	1380	+ throw new Exception('Too many consecutive embedding failures detected. Please check your API configuration.');
	1381	+ }
	1382	+ }
	1383	+
	1384	+ // If all URLs in this batch failed, stop processing
	1385	+ if ($batch_stats['processed'] === 0 && $batch_stats['failed'] > 0) {
	1386	+ $status['status'] = 'error';
	1387	+ $status['error'] = sprintf(
	1388	+ 'Processing stopped: %d consecutive failures in batch. Last error: %s',
	1389	+ $batch_stats['failed'],
	1390	+ $batch_stats['last_error']
	1391	+ );
	1392	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1393	+ return;
	1394	+ }
	1395	+
	1396	+ // Update final progress
	1397	+ $status['processed_urls'] = min($end_url, $total_urls);
	1398	+ $status['last_update'] = time();
	1399	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1400	+
	1401	+ // Check if we've processed all URLs
	1402	+ if ($end_url >= $total_urls) {
	1403	+ // All URLs have been processed - mark as complete
	1404	+ $status['status'] = 'complete';
	1405	+ $status['processed_urls'] = $total_urls;
	1406	+
	1407	+ // Add completion summary
	1408	+ $status['completion_summary'] = [
	1409	+ 'total_urls' => $total_urls,
	1410	+ 'successful_urls' => $total_urls - absint($status['failed_urls'] ?? 0),
	1411	+ 'failed_urls' => absint($status['failed_urls'] ?? 0),
	1412	+ 'completion_time' => current_time('mysql')
	1413	+ ];
	1414	+
	1415	+ // Save the completed status (don't delete it - let user dismiss manually)
	1416	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1417	+
	1418	+ // DON'T delete the status transients here - let user dismiss manually
	1419	+ } else {
	1420	+ // Schedule next batch
	1421	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_sitemap_urls', array(
	1422	+ 'urls' => $urls,
	1423	+ 'sitemap_url' => $sitemap_url,
	1424	+ 'total_urls' => $total_urls,
	1425	+ 'batch_size' => $batch_size,
	1426	+ 'batch_pause' => $batch_pause,
	1427	+ ));
	1428	+ }
	1429	+ } catch (\Exception $e) {
	1430	+ $status['status'] = 'error';
	1431	+ $status['error'] = $e->getMessage();
	1432	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1433	+ }
1228	1434	}
1229		-
1230	1435	public function mxchat_sanitize_content_for_api($content) {
1231	1436	//error_log('[MXCHAT-SANITIZE] Original content preview: ' . substr($content, 0, 500) . '...');
1232		-
1233		- // Remove shortcode tags but PRESERVE content inside them
1234		- $content = $this->strip_shortcode_tags_preserve_content($content);
1235		-
	1437	+
1236	1438	// Remove script, style tags, and HTML comments
1237	1439	$content = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $content);
1238	1440	$content = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $content);
1239	1441	$content = preg_replace('/<!--(.|\s)*?-->/', '', $content);
1240		-
	1442	+
1241	1443	// Remove all HTML tags and decode HTML entities
1242	1444	$content = wp_strip_all_tags($content);
1243	1445	$content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5);
1244	1446
		@@ -1296,12 +1498,12 @@
1296	1498	@$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'), LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
1297	1499	$xpath = new DOMXPath($dom);
1298	1500
1299	1501	// For debugging purposes
1300		- $debugEnabled = true; // Set to true to enable debugging output
	1502	+ $debugEnabled = false; // Set to true to enable debugging output
1301	1503	$debug = function($message) use ($debugEnabled) {
1302	1504	if ($debugEnabled) {
1303		- //error_log('[MXCHAT-EXTRACT-DEBUG] ' . $message);
	1505	+ //error_log('[MXCHAT-DEBUG] ' . $message);
1304	1506	}
1305	1507	};
1306	1508
1307	1509	// Direct targeting for Gerow theme posts
		@@ -1400,25 +1602,18 @@
1400	1602	'//main',
1401	1603	'//div[contains(@class, "content")]'
1402	1604	];
1403	1605
1404		- // First handle Elementor content - get only leaf widget containers to avoid duplicates
	1606	+ // First handle Elementor content
1405	1607	$debug("Checking for Elementor content");
1406		- // Get widget containers that are direct children of widgets (not nested inside other widget containers)
1407		- $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-widget")]//div[contains(@class, "elementor-widget-container")]');
	1608	+ $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-element")]//div[contains(@class, "elementor-widget-container")]');
1408	1609	if ($elementor_widgets && $elementor_widgets->length > 0) {
1409	1610	$debug("Found Elementor widgets");
1410		- $seen_content = array(); // Track seen content to avoid duplicates
1411	1611	$combined_content = '';
1412	1612	foreach ($elementor_widgets as $widget) {
1413	1613	$widget_content = $dom->saveHTML($widget);
1414	1614	if (!empty($widget_content)) {
1415		- // Create a hash of the content to detect duplicates
1416		- $content_hash = md5($widget_content);
1417		- if (!isset($seen_content[$content_hash])) {
1418		- $seen_content[$content_hash] = true;
1419		- $combined_content .= $widget_content;
1420		- }
	1615	+ $combined_content .= $widget_content;
1421	1616	}
1422	1617	}
1423	1618	if (!empty($combined_content)) {
1424	1619	$debug("Returning Elementor content");
		@@ -1430,14 +1625,15 @@
1430	1625	foreach ($selectors as $selector) {
1431	1626	$debug("Trying selector: " . $selector);
1432	1627	$nodes = $xpath->query($selector);
1433	1628	if ($nodes && $nodes->length > 0) {
1434		- $debug("Found " . $nodes->length . " matches for selector: " . $selector);
1435		- // Only take the FIRST matching node to avoid duplicate content
1436		- // (pages often have nested or multiple containers with same class)
1437		- $content = $dom->saveHTML($nodes->item(0));
	1629	+ $debug("Found matches for selector: " . $selector);
	1630	+ $content = '';
	1631	+ foreach ($nodes as $node) {
	1632	+ $content .= $dom->saveHTML($node);
	1633	+ }
1438	1634	if (!empty($content)) {
1439		- $debug("Returning content from selector: " . $selector . " (first match only)");
	1635	+ $debug("Returning content from selector: " . $selector);
1440	1636	return $content;
1441	1637	}
1442	1638	}
1443	1639	}
		@@ -1461,106 +1657,16 @@
1461	1657	$debug("Returning blog-area section content");
1462	1658	return $content;
1463	1659	}
1464	1660	}
1465		-
1466		- // Generic container selectors for non-CMS sites (like .asp pages)
1467		- $debug("Trying generic container selectors");
1468		- $generic_selectors = [
1469		- '//div[@id="main"]',
1470		- '//div[@id="wrapper"]',
1471		- '//div[@id="page"]',
1472		- '//div[@id="site-content"]',
1473		- '//div[contains(@class, "main-content")]',
1474		- '//div[contains(@class, "page-content")]',
1475		- '//div[contains(@class, "site-content")]',
1476		- ];
1477		-
1478		- foreach ($generic_selectors as $selector) {
1479		- $debug("Trying generic selector: " . $selector);
1480		- $nodes = $xpath->query($selector);
1481		- if ($nodes && $nodes->length > 0) {
1482		- $content = $dom->saveHTML($nodes->item(0));
1483		- if (!empty($content)) {
1484		- $debug("Returning content from generic selector: " . $selector);
1485		- return $content;
1486		- }
1487		- }
1488		- }
1489		-
1490		- // Paragraph-based content detection - find regions with substantial text
1491		- $debug("Trying paragraph-based content detection");
1492		- $paragraphs = $xpath->query('//p[string-length(normalize-space()) > 50]');
1493		- if ($paragraphs && $paragraphs->length >= 3) {
1494		- $debug("Found " . $paragraphs->length . " substantial paragraphs");
1495		- // Collect all substantial paragraphs and their content
1496		- $paragraph_content = '';
1497		- foreach ($paragraphs as $p) {
1498		- $paragraph_content .= $dom->saveHTML($p) . "\n";
1499		- }
1500		- if (!empty($paragraph_content)) {
1501		- $debug("Returning paragraph-based content");
1502		- return $paragraph_content;
1503		- }
1504		- }
1505		-
1506		- // Improved body fallback - strip nav/header/footer elements first
1507		- $debug("Using improved body fallback");
	1661	+
	1662	+ // Fallback: Return the body content if no specific selector matches
	1663	+ $debug("Using body fallback");
1508	1664	$body = $dom->getElementsByTagName('body');
1509	1665	if ($body->length > 0) {
1510		- // Clone the body to avoid modifying the original DOM
1511		- $body_clone = $body->item(0)->cloneNode(true);
1512		-
1513		- // Remove common non-content elements by tag name
1514		- $remove_tags = ['nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript'];
1515		- foreach ($remove_tags as $tag) {
1516		- $elements = $body_clone->getElementsByTagName($tag);
1517		- // Iterate backwards to safely remove elements
1518		- for ($i = $elements->length - 1; $i >= 0; $i--) {
1519		- $el = $elements->item($i);
1520		- if ($el && $el->parentNode) {
1521		- $el->parentNode->removeChild($el);
1522		- }
1523		- }
1524		- }
1525		-
1526		- // Remove elements with common non-content class names using XPath on the cloned body
1527		- $temp_dom = new DOMDocument();
1528		- @$temp_dom->appendChild($temp_dom->importNode($body_clone, true));
1529		- $temp_xpath = new DOMXPath($temp_dom);
1530		-
1531		- $remove_class_patterns = [
1532		- '//*[contains(@class, "nav")]',
1533		- '//*[contains(@class, "menu")]',
1534		- '//*[contains(@class, "sidebar")]',
1535		- '//*[contains(@class, "footer")]',
1536		- '//*[contains(@class, "header")]',
1537		- '//*[contains(@id, "nav")]',
1538		- '//*[contains(@id, "menu")]',
1539		- '//*[contains(@id, "sidebar")]',
1540		- '//*[contains(@id, "footer")]',
1541		- '//*[contains(@id, "header")]',
1542		- ];
1543		-
1544		- foreach ($remove_class_patterns as $pattern) {
1545		- $elements = $temp_xpath->query($pattern);
1546		- if ($elements) {
1547		- for ($i = $elements->length - 1; $i >= 0; $i--) {
1548		- $el = $elements->item($i);
1549		- if ($el && $el->parentNode) {
1550		- $el->parentNode->removeChild($el);
1551		- }
1552		- }
1553		- }
1554		- }
1555		-
1556		- $cleaned_content = $temp_dom->saveHTML();
1557		- if (!empty($cleaned_content)) {
1558		- $debug("Returning cleaned body content");
1559		- return $cleaned_content;
1560		- }
	1666	+ return $dom->saveHTML($body->item(0));
1561	1667	}
1562		-
	1668	+
1563	1669	// Last resort: return the original HTML
1564	1670	$debug("Returning original HTML");
1565	1671	return $html;
1566	1672	} catch (Exception $e) {
		@@ -1612,41 +1718,47 @@
1612	1718	try {
1613	1719	// Verify the request
1614	1720	check_ajax_referer('mxchat_status_nonce', 'nonce');
1615	1721
1616		- // Get active queue IDs
1617		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
1618		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
	1722	+ // Get the status just like in your admin page
	1723	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1724	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
1619	1725
1620		- $sitemap_status = false;
1621		- $pdf_status = false;
	1726	+ $pdf_status = $pdf_url ? $this->mxchat_get_pdf_processing_status($pdf_url) : false;
	1727	+ $sitemap_status = $sitemap_url ? $this->mxchat_get_sitemap_processing_status($sitemap_url) : false;
1622	1728
1623		- // Get sitemap queue status
1624		- if ($sitemap_queue_id) {
1625		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
	1729	+ // Add the PDF URL to the status object
	1730	+ if ($pdf_status && $pdf_url) {
	1731	+ $pdf_status['pdf_url'] = $pdf_url;
1626	1732	}
1627	1733
1628		- // Get PDF queue status
1629		- if ($pdf_queue_id) {
1630		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
1631		- }
	1734	+ // Set the current PDF URL for the manual batch processing button
	1735	+ $current_pdf_url = $pdf_url;
1632	1736
	1737	+ // Check for true processing status, not just presence of status
1633	1738	$is_active_processing =
1634		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
1635		- ($pdf_status && $pdf_status['status'] === 'processing');
	1739	+ ($sitemap_status && isset($sitemap_status['status']) && $sitemap_status['status'] === 'processing') ||
	1740	+ ($pdf_status && isset($pdf_status['status']) && $pdf_status['status'] === 'processing');
1636	1741
	1742	+ // Get single URL status, but only if no processing is active
	1743	+ $single_url_status = !$is_active_processing ? $this->mxchat_get_single_url_status() : false;
	1744	+
	1745	+ // REMOVED: Auto-clearing of completed status - now only done via dismiss button
	1746	+
1637	1747	// Return JSON response with the status data
1638	1748	wp_send_json(array(
1639	1749	'pdf_status' => $pdf_status,
1640	1750	'sitemap_status' => $sitemap_status,
	1751	+ 'single_url_status' => $single_url_status,
1641	1752	'is_processing' => $is_active_processing,
1642		- 'sitemap_queue_id' => $sitemap_queue_id,
1643		- 'pdf_queue_id' => $pdf_queue_id
	1753	+ 'current_pdf_url' => $current_pdf_url
1644	1754	));
1645	1755
1646	1756	} catch (Exception $e) {
	1757	+ // Log the error
1647	1758	//error_log('MxChat Status Update Error: ' . $e->getMessage());
1648	1759
	1760	+ // Return a friendly error response
1649	1761	wp_send_json_error(array(
1650	1762	'message' => 'Error getting status updates: ' . $e->getMessage(),
1651	1763	'status' => 'error'
1652	1764	));
		@@ -1651,1373 +1763,8 @@
1651	1763	'status' => 'error'
1652	1764	));
1653	1765	}
1654	1766	}
1655		-
1656		-/**
1657		- * Helper function to get queue status data
1658		- */
1659		-private function mxchat_get_queue_status_data($queue_id, $type = 'sitemap') {
1660		- global $wpdb;
1661		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
1662		-
1663		- // Get counts by status
1664		- $counts = $wpdb->get_results($wpdb->prepare(
1665		- "SELECT status, COUNT(*) as count
1666		- FROM $table_name
1667		- WHERE queue_id = %s
1668		- GROUP BY status",
1669		- $queue_id
1670		- ), OBJECT_K);
1671		-
1672		- $total = 0;
1673		- $completed = 0;
1674		- $failed = 0;
1675		- $processing = 0;
1676		- $pending = 0;
1677		-
1678		- foreach ($counts as $status => $data) {
1679		- $count = absint($data->count);
1680		- $total += $count;
1681		-
1682		- switch ($status) {
1683		- case 'completed':
1684		- $completed = $count;
1685		- break;
1686		- case 'failed':
1687		- $failed = $count;
1688		- break;
1689		- case 'processing':
1690		- $processing = $count;
1691		- break;
1692		- case 'pending':
1693		- $pending = $count;
1694		- break;
1695		- }
1696		- }
1697		-
1698		- if ($total === 0) {
1699		- return false;
1700		- }
1701		-
1702		- // Calculate percentage
1703		- $percentage = round((($completed + $failed) / $total) * 100);
1704		-
1705		- // Get failed items details (limit to 50)
1706		- $failed_items = array();
1707		- if ($failed > 0) {
1708		- $failed_results = $wpdb->get_results($wpdb->prepare(
1709		- "SELECT item_type, item_data, error_message, attempts, completed_at
1710		- FROM $table_name
1711		- WHERE queue_id = %s
1712		- AND status = 'failed'
1713		- AND attempts >= max_attempts
1714		- ORDER BY id DESC
1715		- LIMIT 50",
1716		- $queue_id
1717		- ));
1718		-
1719		- foreach ($failed_results as $item) {
1720		- $data = json_decode($item->item_data, true);
1721		- $url = $type === 'pdf' ? ($data['pdf_url'] ?? '') : ($data['url'] ?? '');
1722		-
1723		- $failed_items[] = array(
1724		- 'url' => $url,
1725		- 'page' => $type === 'pdf' ? ($data['page_number'] ?? 0) : 0,
1726		- 'error' => $item->error_message,
1727		- 'retries' => $item->attempts,
1728		- 'time' => strtotime($item->completed_at)
1729		- );
1730		- }
1731		- }
1732		-
1733		- // Get queue metadata
1734		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
1735		-
1736		- // Determine if queue is complete
1737		- $is_complete = ($pending === 0 && $processing === 0);
1738		-
1739		- // Get last update time
1740		- $last_update = $wpdb->get_var($wpdb->prepare(
1741		- "SELECT MAX(UNIX_TIMESTAMP(COALESCE(completed_at, started_at, created_at)))
1742		- FROM $table_name
1743		- WHERE queue_id = %s",
1744		- $queue_id
1745		- ));
1746		-
1747		- $last_update_text = $last_update ? human_time_diff($last_update, time()) . ' ' . __('ago', 'mxchat') : __('Just now', 'mxchat');
1748		-
1749		- // Format based on type
1750		- if ($type === 'pdf') {
1751		- return array(
1752		- 'total_pages' => $total,
1753		- 'processed_pages' => $completed + $failed,
1754		- 'failed_pages' => $failed,
1755		- 'percentage' => $percentage,
1756		- 'status' => $is_complete ? 'complete' : 'processing',
1757		- 'last_update' => $last_update_text,
1758		- 'failed_pages_list' => $failed_items,
1759		- 'pdf_url' => $source_url,
1760		- 'queue_id' => $queue_id
1761		- );
1762		- } else {
1763		- return array(
1764		- 'total_urls' => $total,
1765		- 'processed_urls' => $completed + $failed,
1766		- 'failed_urls' => $failed,
1767		- 'percentage' => $percentage,
1768		- 'status' => $is_complete ? 'complete' : 'processing',
1769		- 'last_update' => $last_update_text,
1770		- 'failed_urls_list' => $failed_items,
1771		- 'sitemap_url' => $source_url,
1772		- 'queue_id' => $queue_id
1773		- );
1774		- }
1775		-}
1776		-
1777		-/**
1778		- * Public method to get processing status for both sitemap and PDF queues
1779		- * Used by admin pages to display processing status
1780		- *
1781		- * @return array Array with 'sitemap_status', 'pdf_status', and 'is_processing' keys
1782		- */
1783		-public function mxchat_get_processing_statuses() {
1784		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
1785		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
1786		-
1787		- $sitemap_status = false;
1788		- $pdf_status = false;
1789		-
1790		- if ($sitemap_queue_id) {
1791		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
1792		- }
1793		-
1794		- if ($pdf_queue_id) {
1795		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
1796		- }
1797		-
1798		- $is_processing =
1799		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
1800		- ($pdf_status && $pdf_status['status'] === 'processing');
1801		-
1802		- return array(
1803		- 'sitemap_status' => $sitemap_status,
1804		- 'pdf_status' => $pdf_status,
1805		- 'is_processing' => $is_processing
1806		- );
1807		-}
1808		-
1809		-/**
1810		- * AJAX handler to get recent knowledge entries for real-time table updates
1811		- * UPDATED: Now supports both WordPress DB and Pinecone data sources
1812		- */
1813		-public function ajax_mxchat_get_recent_entries() {
1814		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
1815		-
1816		- if (!current_user_can('manage_options')) {
1817		- wp_send_json_error(array('message' => 'Unauthorized'));
1818		- return;
1819		- }
1820		-
1821		- global $wpdb;
1822		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1823		-
1824		- // Get parameters
1825		- $last_id = isset($_POST['last_id']) ? absint($_POST['last_id']) : 0;
1826		- $limit = isset($_POST['limit']) ? min(absint($_POST['limit']), 50) : 10;
1827		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
1828		-
1829		- // Check if Pinecone is enabled for this bot
1830		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
1831		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
1832		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
1833		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
1834		-
1835		- if ($use_pinecone && $has_pinecone_api) {
1836		- // PINECONE DATA SOURCE - Get grouped entry count (not raw vector count)
1837		- // Use mxchat_fetch_pinecone_records which returns total_unique_entries
1838		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, '', 1, 10, $bot_id, '');
1839		- $total_count = $records['total'] ?? 0;
1840		-
1841		- // For Pinecone, we don't return individual entries during polling
1842		- // (entries are already displayed on page load via mxchat_fetch_pinecone_records)
1843		- // We just return the updated count
1844		- wp_send_json_success(array(
1845		- 'entries' => array(),
1846		- 'total_count' => absint($total_count),
1847		- 'max_id' => $last_id,
1848		- 'data_source' => 'pinecone'
1849		- ));
1850		- return;
1851		- }
1852		-
1853		- // WORDPRESS DB DATA SOURCE
1854		- // Build query to get entries newer than last_id
1855		- $where_clauses = array('1=1');
1856		- $where_values = array();
1857		-
1858		- if ($last_id > 0) {
1859		- $where_clauses[] = 'id > %d';
1860		- $where_values[] = $last_id;
1861		- }
1862		-
1863		- // Note: WordPress DB table doesn't have bot_id column
1864		- // Multi-bot filtering is handled via Pinecone namespaces
1865		-
1866		- $where_sql = implode(' AND ', $where_clauses);
1867		-
1868		- // Get recent entries
1869		- $query = "SELECT id, article_content, source_url, timestamp
1870		- FROM $table_name
1871		- WHERE $where_sql
1872		- ORDER BY id DESC
1873		- LIMIT %d";
1874		-
1875		- $where_values[] = $limit;
1876		-
1877		- $entries = $wpdb->get_results($wpdb->prepare($query, $where_values));
1878		-
1879		- // Get total count of GROUPED entries (by source_url) - matches pagination display
1880		- // Count unique source_urls (excluding mxchat:// internal refs) + count of ungrouped rows
1881		- $total_count = $wpdb->get_var(
1882		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
1883		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
1884		- );
1885		-
1886		- // Format entries for response
1887		- $formatted_entries = array();
1888		- $preview_length = 150;
1889		- foreach ($entries as $entry) {
1890		- // Parse chunk metadata using the proper chunker method (same as initial page load)
1891		- if (class_exists('MxChat_Chunker')) {
1892		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($entry->article_content);
1893		- $display_content = $chunk_meta['text'];
1894		- $chunk_metadata = $chunk_meta['metadata'];
1895		- } else {
1896		- $display_content = $entry->article_content;
1897		- $chunk_metadata = array();
1898		- }
1899		-
1900		- $content_preview = mb_strlen($display_content) > $preview_length
1901		- ? mb_substr($display_content, 0, $preview_length) . '...'
1902		- : $display_content;
1903		-
1904		- $formatted_entries[] = array(
1905		- 'id' => $entry->id,
1906		- 'preview' => esc_html($content_preview),
1907		- 'full_content' => wp_kses_post(wpautop($display_content)),
1908		- 'content_length' => mb_strlen($display_content),
1909		- 'preview_length' => $preview_length,
1910		- 'source_url' => $entry->source_url,
1911		- 'has_link' => !empty($entry->source_url) && strpos($entry->source_url, 'mxchat://') !== 0,
1912		- 'chunk_metadata' => $chunk_metadata,
1913		- 'bot_id' => $entry->bot_id ?? 'default',
1914		- 'edit_nonce' => wp_create_nonce('mxchat_edit_entry_nonce'),
1915		- 'delete_nonce' => wp_create_nonce('mxchat_delete_prompt_nonce')
1916		- );
1917		- }
1918		-
1919		- wp_send_json_success(array(
1920		- 'entries' => $formatted_entries,
1921		- 'total_count' => absint($total_count),
1922		- 'max_id' => !empty($entries) ? $entries[0]->id : $last_id,
1923		- 'data_source' => 'wordpress'
1924		- ));
1925		-}
1926		-
1927		-/**
1928		- * Get Pinecone total count from stats API
1929		- * Helper function for ajax_mxchat_get_recent_entries
1930		- */
1931		-private function mxchat_get_pinecone_count_from_stats($pinecone_options) {
1932		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1933		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1934		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1935		-
1936		- if (empty($api_key) || empty($host)) {
1937		- return 0;
1938		- }
1939		-
1940		- try {
1941		- $stats_url = "https://{$host}/describe_index_stats";
1942		-
1943		- $response = wp_remote_post($stats_url, array(
1944		- 'headers' => array(
1945		- 'Api-Key' => $api_key,
1946		- 'Content-Type' => 'application/json'
1947		- ),
1948		- 'body' => '{}',
1949		- 'timeout' => 10
1950		- ));
1951		-
1952		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
1953		- $body = wp_remote_retrieve_body($response);
1954		- $stats_data = json_decode($body, true);
1955		-
1956		- // If namespace is specified, get count from that specific namespace
1957		- if (!empty($namespace) && isset($stats_data['namespaces'][$namespace]['vectorCount'])) {
1958		- return intval($stats_data['namespaces'][$namespace]['vectorCount']);
1959		- }
1960		-
1961		- // If no namespace specified or namespace not found in response, use total
1962		- return intval($stats_data['totalVectorCount'] ?? 0);
1963		- }
1964		-
1965		- return 0;
1966		-
1967		- } catch (Exception $e) {
1968		- return 0;
1969		- }
1970		-}
1971		-
1972		-/**
1973		- * AJAX handler to refresh Pinecone entries table via AJAX
1974		- * Returns the table HTML for updating the UI without a full page reload
1975		- */
1976		-public function ajax_mxchat_refresh_pinecone_entries() {
1977		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
1978		-
1979		- if (!current_user_can('manage_options')) {
1980		- wp_send_json_error(array('message' => 'Unauthorized'));
1981		- return;
1982		- }
1983		-
1984		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
1985		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
1986		- $per_page = 25;
1987		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
1988		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
1989		-
1990		- // Get Pinecone manager and options
1991		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
1992		- if (!$pinecone_manager) {
1993		- wp_send_json_error(array('message' => 'Pinecone manager not available'));
1994		- return;
1995		- }
1996		-
1997		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
1998		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
1999		- $pinecone_api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2000		-
2001		- if (!$use_pinecone || empty($pinecone_api_key)) {
2002		- wp_send_json_error(array('message' => 'Pinecone not configured'));
2003		- return;
2004		- }
2005		-
2006		- // Fetch records from Pinecone
2007		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, $search_query, $page, $per_page, $bot_id, $content_type_filter);
2008		- $prompts = $records['data'] ?? array();
2009		- $total_records = $records['total'] ?? 0;
2010		-
2011		- // Preprocess Pinecone records — set chunk_metadata and display_content
2012		- // (matches admin-knowledge-page.php preprocessing)
2013		- foreach ($prompts as $prompt) {
2014		- if (isset($prompt->chunk_index) && $prompt->chunk_index !== null) {
2015		- $prompt->chunk_metadata = array(
2016		- 'chunk_index' => intval($prompt->chunk_index),
2017		- 'total_chunks' => isset($prompt->total_chunks) ? intval($prompt->total_chunks) : null,
2018		- 'is_chunked' => isset($prompt->is_chunked) ? (bool) $prompt->is_chunked : true,
2019		- 'source_url' => $prompt->source_url ?? ''
2020		- );
2021		- $prompt->display_content = $prompt->article_content;
2022		- } else {
2023		- $prompt->chunk_metadata = array();
2024		- $prompt->display_content = $prompt->article_content ?? '';
2025		- }
2026		- }
2027		-
2028		- // Group prompts by source_url
2029		- $grouped_prompts = array();
2030		- foreach ($prompts as $prompt) {
2031		- $source_url = '';
2032		- if (!empty($prompt->chunk_metadata['source_url'])) {
2033		- $source_url = $prompt->chunk_metadata['source_url'];
2034		- } elseif (!empty($prompt->source_url)) {
2035		- $source_url = $prompt->source_url;
2036		- }
2037		-
2038		- if (!empty($source_url)) {
2039		- if (!isset($grouped_prompts[$source_url])) {
2040		- $grouped_prompts[$source_url] = array();
2041		- }
2042		- $grouped_prompts[$source_url][] = $prompt;
2043		- } else {
2044		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2045		- }
2046		- }
2047		-
2048		- // Sort each group by chunk_index
2049		- foreach ($grouped_prompts as $source_url => &$group) {
2050		- usort($group, function($a, $b) {
2051		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2052		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2053		- return $index_a - $index_b;
2054		- });
2055		- }
2056		- unset($group);
2057		-
2058		- // Build HTML for the table rows - must match admin-knowledge-page.php structure exactly
2059		- ob_start();
2060		- $display_index = 0;
2061		- $current_page = $page;
2062		- $data_source = 'pinecone';
2063		- $current_bot_id = $bot_id;
2064		- $preview_length = 150;
2065		-
2066		- if (empty($grouped_prompts)) {
2067		- echo '<tr><td colspan="4" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2068		- esc_html_e('No knowledge entries found in Pinecone.', 'mxchat');
2069		- echo '</td></tr>';
2070		- } else {
2071		- foreach ($grouped_prompts as $source_url => $group) {
2072		- $chunk_count = count($group);
2073		- $first_prompt = $group[0];
2074		- $display_index++;
2075		-
2076		- if ($chunk_count > 1) {
2077		- // Multiple chunks - show grouped row with expand button
2078		- $group_id = 'group-' . md5($source_url);
2079		- ?>
2080		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2081		- class="mxchat-chunk-group-header"
2082		- data-source="<?php echo esc_attr($data_source); ?>"
2083		- data-group-id="<?php echo esc_attr($group_id); ?>"
2084		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2085		- <td style="padding: 12px 16px; text-align: center;">
2086		- <input type="checkbox"
2087		- class="mxchat-entry-checkbox"
2088		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2089		- data-source="<?php echo esc_attr($data_source); ?>"
2090		- data-source-url="<?php echo esc_attr($source_url); ?>"
2091		- data-is-group="true"
2092		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2093		- </td>
2094		- <td style="padding: 12px 16px; font-size: 13px;">
2095		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2096		- </td>
2097		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2098		- <div class="mxchat-chunk-group-info">
2099		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2100		- <span class="dashicons dashicons-arrow-right-alt2"></span>
2101		- </button>
2102		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2103		- <span class="mxchat-chunk-preview">
2104		- <?php
2105		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : (isset($first_prompt->article_content) ? $first_prompt->article_content : '');
2106		- $content_preview = mb_substr($parent_content, 0, 100);
2107		- echo esc_html($content_preview . '...');
2108		- ?>
2109		- </span>
2110		- </div>
2111		- </td>
2112		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2113		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2114		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2115		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2116		- <?php esc_html_e('View Source', 'mxchat'); ?>
2117		- </a>
2118		- <?php else : ?>
2119		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2120		- <?php endif; ?>
2121		- </td>
2122		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2123		- <?php if ($data_source !== 'pinecone') : ?>
2124		- <button type="button"
2125		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2126		- data-source-url="<?php echo esc_attr($source_url); ?>"
2127		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2128		- data-data-source="<?php echo esc_attr($data_source); ?>"
2129		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2130		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2131		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2132		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2133		- </button>
2134		- <?php endif; ?>
2135		- <button type="button"
2136		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2137		- data-source-url="<?php echo esc_attr($source_url); ?>"
2138		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2139		- data-data-source="<?php echo esc_attr($data_source); ?>"
2140		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2141		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2142		- style="color: var(--mxch-error);"
2143		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2144		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2145		- </button>
2146		- </td>
2147		- </tr>
2148		- <?php
2149		- // Render hidden chunk rows
2150		- foreach ($group as $chunk_index => $chunk) {
2151		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
2152		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
2153		- $content = isset($chunk->display_content) ? $chunk->display_content : (isset($chunk->article_content) ? $chunk->article_content : '');
2154		- $content_preview = mb_strlen($content) > $preview_length
2155		- ? mb_substr($content, 0, $preview_length) . '...'
2156		- : $content;
2157		- ?>
2158		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
2159		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
2160		- data-source="<?php echo esc_attr($data_source); ?>"
2161		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
2162		- <td style="padding: 12px 16px; text-align: center;">
2163		- <!-- Checkbox column placeholder for chunks (managed by group) -->
2164		- </td>
2165		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
2166		- <!-- Hidden ID column for chunks -->
2167		- </td>
2168		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2169		- <div class="mxchat-accordion-wrapper">
2170		- <div class="mxchat-content-preview">
2171		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
2172		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
2173		- </span>
2174		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2175		- <?php if (mb_strlen($content) > $preview_length) : ?>
2176		- <button class="mxchat-expand-toggle" type="button">
2177		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2178		- </button>
2179		- <?php endif; ?>
2180		- </div>
2181		- <div class="mxchat-content-full" style="display: none;">
2182		- <div class="content-view">
2183		- <?php
2184		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2185		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2186		- echo wp_kses_post(wpautop($content));
2187		- echo '</div>';
2188		- } else {
2189		- echo wp_kses_post(wpautop($content));
2190		- }
2191		- ?>
2192		- </div>
2193		- </div>
2194		- </div>
2195		- </td>
2196		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2197		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
2198		- </td>
2199		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
2200		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
2201		- </td>
2202		- </tr>
2203		- <?php
2204		- }
2205		- } else {
2206		- // Single entry - display normally with accordion
2207		- $prompt = $first_prompt;
2208		- $content = isset($prompt->display_content) ? $prompt->display_content : (isset($prompt->article_content) ? $prompt->article_content : '');
2209		- $content_preview = mb_strlen($content) > $preview_length
2210		- ? mb_substr($content, 0, $preview_length) . '...'
2211		- : $content;
2212		- ?>
2213		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
2214		- data-source="<?php echo esc_attr($data_source); ?>"
2215		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2216		- <td style="padding: 12px 16px; text-align: center;">
2217		- <input type="checkbox"
2218		- class="mxchat-entry-checkbox"
2219		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2220		- data-source="<?php echo esc_attr($data_source); ?>"
2221		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
2222		- data-is-group="false"
2223		- data-chunk-count="1">
2224		- </td>
2225		- <td style="padding: 12px 16px; font-size: 13px;">
2226		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2227		- </td>
2228		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2229		- <div class="mxchat-accordion-wrapper">
2230		- <div class="mxchat-content-preview">
2231		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2232		- <?php if (mb_strlen($content) > $preview_length) : ?>
2233		- <button class="mxchat-expand-toggle" type="button">
2234		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2235		- </button>
2236		- <?php endif; ?>
2237		- </div>
2238		- <div class="mxchat-content-full" style="display: none;">
2239		- <div class="content-view">
2240		- <?php
2241		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2242		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2243		- echo wp_kses_post(wpautop($content));
2244		- echo '</div>';
2245		- } else {
2246		- echo wp_kses_post(wpautop($content));
2247		- }
2248		- ?>
2249		- </div>
2250		- </div>
2251		- </div>
2252		- </td>
2253		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2254		- <?php
2255		- $actual_source = $source_url;
2256		- if (strpos($source_url, '_ungrouped_') === 0) {
2257		- $actual_source = $prompt->source_url ?? '';
2258		- }
2259		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
2260		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2261		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2262		- <?php esc_html_e('View', 'mxchat'); ?>
2263		- </a>
2264		- <?php else : ?>
2265		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
2266		- <?php endif; ?>
2267		- </td>
2268		- <td style="padding: 12px 16px;">
2269		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-ajax" data-vector-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_pinecone_prompt_nonce'); ?>" style="color: var(--mxch-error);">
2270		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2271		- </button>
2272		- </td>
2273		- </tr>
2274		- <?php
2275		- }
2276		- }
2277		- }
2278		- $html = ob_get_clean();
2279		-
2280		- // Generate pagination HTML for Pinecone
2281		- $total_pages = ceil($total_records / $per_page);
2282		- $pagination_html = '';
2283		- if ($total_pages > 1) {
2284		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
2285		-
2286		- // Previous button
2287		- if ($page > 1) {
2288		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
2289		- }
2290		-
2291		- // Page numbers
2292		- $start_page = max(1, $page - 2);
2293		- $end_page = min($total_pages, $page + 2);
2294		-
2295		- if ($start_page > 1) {
2296		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
2297		- if ($start_page > 2) {
2298		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2299		- }
2300		- }
2301		-
2302		- for ($i = $start_page; $i <= $end_page; $i++) {
2303		- if ($i == $page) {
2304		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
2305		- } else {
2306		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
2307		- }
2308		- }
2309		-
2310		- if ($end_page < $total_pages) {
2311		- if ($end_page < $total_pages - 1) {
2312		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2313		- }
2314		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
2315		- }
2316		-
2317		- // Next button
2318		- if ($page < $total_pages) {
2319		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
2320		- }
2321		-
2322		- $pagination_html .= '</div>';
2323		- }
2324		-
2325		- wp_send_json_success(array(
2326		- 'html' => $html,
2327		- 'pagination_html' => $pagination_html,
2328		- 'total_count' => $total_records,
2329		- 'total_pages' => $total_pages,
2330		- 'page' => $page,
2331		- 'per_page' => $per_page,
2332		- 'data_source' => 'pinecone'
2333		- ));
2334		-}
2335		-
2336		-/**
2337		- * AJAX handler for pagination - handles both WordPress DB and Pinecone data sources
2338		- * Returns paginated entries without requiring a full page reload
2339		- */
2340		-public function ajax_mxchat_paginate_entries() {
2341		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2342		-
2343		- if (!current_user_can('manage_options')) {
2344		- wp_send_json_error(array('message' => 'Unauthorized'));
2345		- return;
2346		- }
2347		-
2348		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2349		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2350		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2351		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2352		- $per_page = 25;
2353		-
2354		- // Check if Pinecone is enabled for this bot
2355		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
2356		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
2357		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2358		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
2359		-
2360		- if ($use_pinecone && $has_pinecone_api) {
2361		- // Delegate to Pinecone pagination handler (pass search params)
2362		- $_POST['page'] = $page;
2363		- $_POST['search'] = $search_query;
2364		- $_POST['content_type'] = $content_type_filter;
2365		- $this->ajax_mxchat_refresh_pinecone_entries();
2366		- return;
2367		- }
2368		-
2369		- // WordPress DB pagination - MUST match initial page load logic exactly
2370		- global $wpdb;
2371		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2372		- $offset = ($page - 1) * $per_page;
2373		-
2374		- // Build WHERE clause for search and content type filtering
2375		- $where_clauses = array();
2376		- $where_values = array();
2377		-
2378		- if ($search_query) {
2379		- $where_clauses[] = "article_content LIKE %s";
2380		- $where_values[] = '%' . $wpdb->esc_like($search_query) . '%';
2381		- }
2382		-
2383		- if ($content_type_filter) {
2384		- switch ($content_type_filter) {
2385		- case 'manual':
2386		- $where_clauses[] = "(source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')";
2387		- break;
2388		- case 'pdf':
2389		- $where_clauses[] = "source_url LIKE '%.pdf'";
2390		- break;
2391		- case 'url':
2392		- $where_clauses[] = "source_url != '' AND source_url IS NOT NULL AND source_url NOT LIKE 'mxchat://%' AND source_url NOT LIKE '%.pdf'";
2393		- break;
2394		- }
2395		- }
2396		-
2397		- $where_sql = !empty($where_clauses) ? 'WHERE ' . implode(' AND ', $where_clauses) : '';
2398		-
2399		- // Count grouped entries with filters applied
2400		- if (!empty($where_values)) {
2401		- $count_args = array_merge($where_values, $where_values);
2402		- $count_query = $wpdb->prepare(
2403		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2404		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))",
2405		- ...$count_args
2406		- );
2407		- $total_records = $wpdb->get_var($count_query);
2408		- } else if (!empty($where_sql)) {
2409		- // Content type filter only (no search), no prepared values needed
2410		- $total_records = $wpdb->get_var(
2411		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2412		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))"
2413		- );
2414		- } else {
2415		- // No filters
2416		- $total_records = $wpdb->get_var(
2417		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2418		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
2419		- );
2420		- }
2421		- $total_pages = ceil($total_records / $per_page);
2422		-
2423		- // Step 1: Get unique source_urls for this page (ordered by latest timestamp) with filters
2424		- if (!empty($where_values)) {
2425		- $query_args = array_merge($where_values, array($per_page, $offset));
2426		- $urls_query = $wpdb->prepare(
2427		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2428		- {$where_sql}
2429		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2430		- ...$query_args
2431		- );
2432		- } else if (!empty($where_sql)) {
2433		- $urls_query = $wpdb->prepare(
2434		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2435		- {$where_sql}
2436		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2437		- $per_page, $offset
2438		- );
2439		- } else {
2440		- $urls_query = $wpdb->prepare(
2441		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2442		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2443		- $per_page, $offset
2444		- );
2445		- }
2446		- $page_urls = $wpdb->get_results($urls_query);
2447		-
2448		- // Step 2: Build list of source_urls to fetch
2449		- $url_list = array();
2450		- $url_order_map = array();
2451		- $order_index = 0;
2452		- foreach ($page_urls as $url_row) {
2453		- $url = $url_row->source_url;
2454		- $url_list[] = $url;
2455		- $url_order_map[$url] = $order_index++;
2456		- }
2457		-
2458		- // Step 3: Fetch all rows for these source_urls (with search filter if applicable)
2459		- $prompts = array();
2460		- if (!empty($url_list)) {
2461		- $placeholders = implode(',', array_fill(0, count($url_list), '%s'));
2462		- if ($search_query) {
2463		- // Include search filter in the final fetch
2464		- $prompts_query = $wpdb->prepare(
2465		- "SELECT id, article_content, source_url, timestamp, role_restriction
2466		- FROM {$table_name}
2467		- WHERE source_url IN ($placeholders) AND article_content LIKE %s
2468		- ORDER BY timestamp DESC",
2469		- ...array_merge($url_list, ['%' . $wpdb->esc_like($search_query) . '%'])
2470		- );
2471		- } else {
2472		- $prompts_query = $wpdb->prepare(
2473		- "SELECT id, article_content, source_url, timestamp, role_restriction
2474		- FROM {$table_name}
2475		- WHERE source_url IN ($placeholders)
2476		- ORDER BY timestamp DESC",
2477		- $url_list
2478		- );
2479		- }
2480		- $prompts = $wpdb->get_results($prompts_query);
2481		- }
2482		-
2483		- // Group prompts by source_url for chunk display
2484		- $grouped_prompts = array();
2485		- foreach ($prompts as $prompt) {
2486		- $source_url = $prompt->source_url ?? '';
2487		-
2488		- // Parse chunk metadata using the proper chunker method (same as initial page load)
2489		- if (class_exists('MxChat_Chunker')) {
2490		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($prompt->article_content);
2491		- $prompt->chunk_metadata = $chunk_meta['metadata'];
2492		- $prompt->display_content = $chunk_meta['text'];
2493		- } else {
2494		- $prompt->chunk_metadata = array();
2495		- $prompt->display_content = $prompt->article_content;
2496		- }
2497		-
2498		- if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0) {
2499		- if (!isset($grouped_prompts[$source_url])) {
2500		- $grouped_prompts[$source_url] = array();
2501		- }
2502		- $grouped_prompts[$source_url][] = $prompt;
2503		- } else {
2504		- // Ungrouped entries
2505		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2506		- }
2507		- }
2508		-
2509		- // Sort groups by the original URL order (newest first)
2510		- uksort($grouped_prompts, function($a, $b) use ($url_order_map) {
2511		- $order_a = isset($url_order_map[$a]) ? $url_order_map[$a] : PHP_INT_MAX;
2512		- $order_b = isset($url_order_map[$b]) ? $url_order_map[$b] : PHP_INT_MAX;
2513		- return $order_a - $order_b;
2514		- });
2515		-
2516		- // Sort each group internally by chunk_index
2517		- foreach ($grouped_prompts as $source_url => &$group) {
2518		- usort($group, function($a, $b) {
2519		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2520		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2521		- return $index_a - $index_b;
2522		- });
2523		- }
2524		- unset($group);
2525		-
2526		- // Build HTML for the table rows
2527		- ob_start();
2528		- $display_index = 0;
2529		- $current_page = $page;
2530		- $data_source = 'wordpress';
2531		- $current_bot_id = $bot_id;
2532		- $preview_length = 150;
2533		-
2534		- if (empty($grouped_prompts)) {
2535		- echo '<tr><td colspan="5" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2536		- esc_html_e('No knowledge entries found. Use the Import Options to add content.', 'mxchat');
2537		- echo '</td></tr>';
2538		- } else {
2539		- foreach ($grouped_prompts as $source_url => $group) {
2540		- $chunk_count = count($group);
2541		- $first_prompt = $group[0];
2542		- $display_index++;
2543		-
2544		- if ($chunk_count > 1) {
2545		- // Multiple chunks - show grouped row with expand button
2546		- $group_id = 'group-' . md5($source_url);
2547		- ?>
2548		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2549		- class="mxchat-chunk-group-header"
2550		- data-source="<?php echo esc_attr($data_source); ?>"
2551		- data-group-id="<?php echo esc_attr($group_id); ?>"
2552		- style="border-bottom: 1px solid var(--mxch-card-border);">
2553		- <td style="padding: 12px 16px; text-align: center;">
2554		- <input type="checkbox"
2555		- class="mxchat-entry-checkbox"
2556		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2557		- data-source="<?php echo esc_attr($data_source); ?>"
2558		- data-source-url="<?php echo esc_attr($source_url); ?>"
2559		- data-is-group="true"
2560		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2561		- </td>
2562		- <td style="padding: 12px 16px; font-size: 13px;">
2563		- <?php echo esc_html($first_prompt->id); ?>
2564		- </td>
2565		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2566		- <div class="mxchat-chunk-group-info">
2567		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2568		- <span class="dashicons dashicons-arrow-right-alt2"></span>
2569		- </button>
2570		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2571		- <span class="mxchat-chunk-preview">
2572		- <?php
2573		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : $first_prompt->article_content;
2574		- $content_preview = mb_substr($parent_content, 0, 100);
2575		- echo esc_html($content_preview . '...');
2576		- ?>
2577		- </span>
2578		- </div>
2579		- </td>
2580		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2581		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2582		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2583		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2584		- <?php esc_html_e('View Source', 'mxchat'); ?>
2585		- </a>
2586		- <?php else : ?>
2587		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2588		- <?php endif; ?>
2589		- </td>
2590		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2591		- <?php if ($data_source !== 'pinecone') : ?>
2592		- <button type="button"
2593		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2594		- data-source-url="<?php echo esc_attr($source_url); ?>"
2595		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2596		- data-data-source="<?php echo esc_attr($data_source); ?>"
2597		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2598		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2599		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2600		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2601		- </button>
2602		- <?php endif; ?>
2603		- <button type="button"
2604		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2605		- data-source-url="<?php echo esc_attr($source_url); ?>"
2606		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2607		- data-data-source="<?php echo esc_attr($data_source); ?>"
2608		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2609		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2610		- style="color: var(--mxch-error);"
2611		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2612		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2613		- </button>
2614		- </td>
2615		- </tr>
2616		- <?php
2617		- // Render hidden chunk rows
2618		- foreach ($group as $chunk_index => $chunk) {
2619		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
2620		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
2621		- $content = isset($chunk->display_content) ? $chunk->display_content : $chunk->article_content;
2622		- $content_preview = mb_strlen($content) > $preview_length
2623		- ? mb_substr($content, 0, $preview_length) . '...'
2624		- : $content;
2625		- ?>
2626		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
2627		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
2628		- data-source="<?php echo esc_attr($data_source); ?>"
2629		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
2630		- <td style="padding: 12px 16px; text-align: center;">
2631		- <!-- Checkbox column placeholder for chunks (managed by group) -->
2632		- </td>
2633		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
2634		- <!-- Hidden ID column for chunks -->
2635		- </td>
2636		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2637		- <div class="mxchat-accordion-wrapper">
2638		- <div class="mxchat-content-preview">
2639		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
2640		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
2641		- </span>
2642		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2643		- <?php if (mb_strlen($content) > $preview_length) : ?>
2644		- <button class="mxchat-expand-toggle" type="button">
2645		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2646		- </button>
2647		- <?php endif; ?>
2648		- </div>
2649		- <div class="mxchat-content-full" style="display: none;">
2650		- <div class="content-view">
2651		- <?php
2652		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2653		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2654		- echo wp_kses_post(wpautop($content));
2655		- echo '</div>';
2656		- } else {
2657		- echo wp_kses_post(wpautop($content));
2658		- }
2659		- ?>
2660		- </div>
2661		- </div>
2662		- </div>
2663		- </td>
2664		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2665		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
2666		- </td>
2667		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
2668		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
2669		- </td>
2670		- </tr>
2671		- <?php
2672		- }
2673		- } else {
2674		- // Single entry - display normally with accordion
2675		- $prompt = $first_prompt;
2676		- $content = isset($prompt->display_content) ? $prompt->display_content : $prompt->article_content;
2677		- $content_preview = mb_strlen($content) > $preview_length
2678		- ? mb_substr($content, 0, $preview_length) . '...'
2679		- : $content;
2680		- ?>
2681		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
2682		- data-source="<?php echo esc_attr($data_source); ?>"
2683		- style="border-bottom: 1px solid var(--mxch-card-border);">
2684		- <td style="padding: 12px 16px; text-align: center;">
2685		- <input type="checkbox"
2686		- class="mxchat-entry-checkbox"
2687		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2688		- data-source="<?php echo esc_attr($data_source); ?>"
2689		- data-source-url="<?php echo esc_attr($source_url); ?>"
2690		- data-is-group="false">
2691		- </td>
2692		- <td style="padding: 12px 16px; font-size: 13px;">
2693		- <?php echo esc_html($prompt->id); ?>
2694		- </td>
2695		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2696		- <div class="mxchat-accordion-wrapper">
2697		- <div class="mxchat-content-preview">
2698		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2699		- <?php if (mb_strlen($content) > $preview_length) : ?>
2700		- <button class="mxchat-expand-toggle" type="button">
2701		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2702		- </button>
2703		- <?php endif; ?>
2704		- </div>
2705		- <div class="mxchat-content-full" style="display: none;">
2706		- <div class="content-view">
2707		- <?php
2708		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2709		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2710		- echo wp_kses_post(wpautop($content));
2711		- echo '</div>';
2712		- } else {
2713		- echo wp_kses_post(wpautop($content));
2714		- }
2715		- ?>
2716		- </div>
2717		- </div>
2718		- </div>
2719		- </td>
2720		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2721		- <?php
2722		- $actual_source = $source_url;
2723		- if (strpos($source_url, '_ungrouped_') === 0) {
2724		- $actual_source = $prompt->source_url ?? '';
2725		- }
2726		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
2727		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2728		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2729		- <?php esc_html_e('View', 'mxchat'); ?>
2730		- </a>
2731		- <?php else : ?>
2732		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
2733		- <?php endif; ?>
2734		- </td>
2735		- <td style="padding: 12px 16px; white-space: nowrap;">
2736		- <button type="button"
2737		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2738		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
2739		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2740		- data-data-source="<?php echo esc_attr($data_source); ?>"
2741		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2742		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2743		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2744		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2745		- </button>
2746		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-wordpress" data-entry-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_wordpress_prompt_nonce'); ?>" style="color: var(--mxch-error);">
2747		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2748		- </button>
2749		- </td>
2750		- </tr>
2751		- <?php
2752		- }
2753		- }
2754		- }
2755		- $html = ob_get_clean();
2756		-
2757		- // Generate pagination HTML (include search/filter data for subsequent pages)
2758		- $pagination_html = '';
2759		- if ($total_pages > 1) {
2760		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
2761		-
2762		- // Previous button
2763		- if ($page > 1) {
2764		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
2765		- }
2766		-
2767		- // Page numbers
2768		- $start_page = max(1, $page - 2);
2769		- $end_page = min($total_pages, $page + 2);
2770		-
2771		- if ($start_page > 1) {
2772		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
2773		- if ($start_page > 2) {
2774		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2775		- }
2776		- }
2777		-
2778		- for ($i = $start_page; $i <= $end_page; $i++) {
2779		- if ($i == $page) {
2780		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
2781		- } else {
2782		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
2783		- }
2784		- }
2785		-
2786		- if ($end_page < $total_pages) {
2787		- if ($end_page < $total_pages - 1) {
2788		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2789		- }
2790		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
2791		- }
2792		-
2793		- // Next button
2794		- if ($page < $total_pages) {
2795		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
2796		- }
2797		-
2798		- $pagination_html .= '</div>';
2799		- }
2800		-
2801		- wp_send_json_success(array(
2802		- 'html' => $html,
2803		- 'pagination_html' => $pagination_html,
2804		- 'total_count' => $total_records,
2805		- 'total_pages' => $total_pages,
2806		- 'page' => $page,
2807		- 'per_page' => $per_page,
2808		- 'data_source' => 'wordpress'
2809		- ));
2810		-}
2811		-
2812		-/**
2813		- * AJAX handler to detect available sitemaps on the site
2814		- * Optimized for speed - only checks primary sitemap indexes first
2815		- */
2816		-public function ajax_mxchat_detect_sitemaps() {
2817		- check_ajax_referer('mxchat_detect_sitemaps_nonce', 'nonce');
2818		-
2819		- if (!current_user_can('manage_options')) {
2820		- wp_send_json_error(array('message' => 'Unauthorized'));
2821		- return;
2822		- }
2823		-
2824		- $site_url = get_site_url();
2825		- $sitemaps = array();
2826		- $found_index = false;
2827		-
2828		- // Only check the main sitemap index files first (much faster)
2829		- // These are the primary entry points that contain sub-sitemaps
2830		- $primary_indexes = array(
2831		- 'sitemap_index.xml' => 'Yoast SEO / Rank Math', // Yoast & Rank Math
2832		- 'wp-sitemap.xml' => 'WordPress Core', // WordPress Core
2833		- 'sitemap.xml' => 'Standard', // Generic/AIOSEO
2834		- );
2835		-
2836		- foreach ($primary_indexes as $path => $source) {
2837		- $url = trailingslashit($site_url) . $path;
2838		-
2839		- $response = wp_remote_head($url, array(
2840		- 'timeout' => 10,
2841		- 'sslverify' => false,
2842		- 'redirection' => 1,
2843		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2844		- ));
2845		-
2846		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2847		- // Found a sitemap index - parse it to get sub-sitemaps
2848		- $sub_sitemaps = $this->parse_sitemap_index($url);
2849		- if (!empty($sub_sitemaps)) {
2850		- $sitemaps[] = array(
2851		- 'url' => $url,
2852		- 'type' => 'index',
2853		- 'source' => $source,
2854		- 'sub_sitemaps' => $sub_sitemaps
2855		- );
2856		- $found_index = true;
2857		- // Found a valid index, no need to check others
2858		- break;
2859		- }
2860		- }
2861		- }
2862		-
2863		- // If no sitemap index found, check for standalone sitemaps
2864		- if (!$found_index) {
2865		- $standalone_sitemaps = array(
2866		- 'post-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
2867		- 'page-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
2868		- );
2869		-
2870		- foreach ($standalone_sitemaps as $path => $info) {
2871		- $url = trailingslashit($site_url) . $path;
2872		-
2873		- $response = wp_remote_head($url, array(
2874		- 'timeout' => 2,
2875		- 'sslverify' => false
2876		- ));
2877		-
2878		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2879		- $sitemaps[] = array(
2880		- 'url' => $url,
2881		- 'type' => $info['type'],
2882		- 'source' => $info['source'],
2883		- 'url_count' => 0 // Skip URL count for speed
2884		- );
2885		- }
2886		- }
2887		- }
2888		-
2889		- wp_send_json_success(array(
2890		- 'sitemaps' => $sitemaps,
2891		- 'site_url' => $site_url
2892		- ));
2893		-}
2894		-
2895		-/**
2896		- * Parse a sitemap index to get sub-sitemaps
2897		- * Optimized: doesn't fetch URL count for each sub-sitemap (too slow)
2898		- */
2899		-private function parse_sitemap_index($url) {
2900		- $sub_sitemaps = array();
2901		-
2902		- $response = wp_remote_get($url, array(
2903		- 'timeout' => 30,
2904		- 'sslverify' => false,
2905		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2906		- 'headers' => array(
2907		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
2908		- 'Accept-Language' => 'en-US,en;q=0.9',
2909		- ),
2910		- ));
2911		-
2912		- if (is_wp_error($response)) {
2913		- return $sub_sitemaps;
2914		- }
2915		-
2916		- $body = wp_remote_retrieve_body($response);
2917		- if (empty($body)) {
2918		- return $sub_sitemaps;
2919		- }
2920		-
2921		- // Suppress XML errors
2922		- libxml_use_internal_errors(true);
2923		- $xml = simplexml_load_string($body);
2924		- libxml_clear_errors();
2925		-
2926		- if ($xml === false) {
2927		- return $sub_sitemaps;
2928		- }
2929		-
2930		- // Check if it's a sitemap index (contains <sitemap> elements)
2931		- if (isset($xml->sitemap)) {
2932		- foreach ($xml->sitemap as $sitemap) {
2933		- $loc = (string) $sitemap->loc;
2934		- if (!empty($loc)) {
2935		- // Try to determine the type from the URL
2936		- $type = 'content';
2937		- if (strpos($loc, 'category') !== false || strpos($loc, 'tag') !== false || strpos($loc, 'taxonomy') !== false) {
2938		- $type = 'taxonomy';
2939		- } elseif (strpos($loc, 'author') !== false || strpos($loc, 'user') !== false) {
2940		- $type = 'author';
2941		- }
2942		-
2943		- // Skip URL count - too slow to fetch for each sitemap
2944		- $sub_sitemaps[] = array(
2945		- 'url' => $loc,
2946		- 'type' => $type,
2947		- 'url_count' => 0, // Don't fetch - takes too long
2948		- 'name' => basename(parse_url($loc, PHP_URL_PATH))
2949		- );
2950		- }
2951		- }
2952		- }
2953		-
2954		- return $sub_sitemaps;
2955		-}
2956		-
2957		-/**
2958		- * Get URL count from a sitemap
2959		- */
2960		-private function get_sitemap_url_count($url) {
2961		- $response = wp_remote_get($url, array(
2962		- 'timeout' => 30,
2963		- 'sslverify' => false,
2964		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2965		- 'headers' => array(
2966		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
2967		- 'Accept-Language' => 'en-US,en;q=0.9',
2968		- ),
2969		- ));
2970		-
2971		- if (is_wp_error($response)) {
2972		- return 0;
2973		- }
2974		-
2975		- $body = wp_remote_retrieve_body($response);
2976		- if (empty($body)) {
2977		- return 0;
2978		- }
2979		-
2980		- // Count <url> or <loc> elements
2981		- $count = preg_match_all('/<url>/i', $body, $matches);
2982		- return $count ?: 0;
2983		-}
2984		-
2985		-/**
2986		- * Get sitemaps declared in robots.txt
2987		- */
2988		-private function get_sitemaps_from_robots($site_url) {
2989		- $sitemaps = array();
2990		- $robots_url = trailingslashit($site_url) . 'robots.txt';
2991		-
2992		- $response = wp_remote_get($robots_url, array(
2993		- 'timeout' => 15,
2994		- 'sslverify' => false,
2995		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2996		- ));
2997		-
2998		- if (is_wp_error($response)) {
2999		- return $sitemaps;
3000		- }
3001		-
3002		- $body = wp_remote_retrieve_body($response);
3003		- if (empty($body)) {
3004		- return $sitemaps;
3005		- }
3006		-
3007		- // Find Sitemap: declarations
3008		- if (preg_match_all('/^Sitemap:\s*(.+)$/mi', $body, $matches)) {
3009		- foreach ($matches[1] as $sitemap_url) {
3010		- $sitemap_url = trim($sitemap_url);
3011		- if (filter_var($sitemap_url, FILTER_VALIDATE_URL)) {
3012		- $sitemaps[] = $sitemap_url;
3013		- }
3014		- }
3015		- }
3016		-
3017		- return $sitemaps;
3018		-}
3019		-
3020	1767	public function mxchat_stop_processing() {
3021	1768	// Verify permissions
3022	1769	if (!current_user_can('manage_options')) {
3023	1770	wp_die(esc_html__('Unauthorized access', 'mxchat'));
		@@ -3025,52 +1772,28 @@
3025	1772
3026	1773	// Verify nonce
3027	1774	check_admin_referer('mxchat_stop_processing_action', 'mxchat_stop_processing_nonce');
3028	1775
3029		- global $wpdb;
3030		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
3031		-
3032		- // Get active queue IDs
3033		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
3034		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
3035		-
3036		- // Delete all pending items from active queues
3037		- if ($sitemap_queue_id) {
3038		- $wpdb->delete(
3039		- $table_name,
3040		- array(
3041		- 'queue_id' => $sitemap_queue_id,
3042		- 'status' => 'pending'
3043		- ),
3044		- array('%s', '%s')
3045		- );
3046		-
3047		- delete_transient('mxchat_active_queue_sitemap');
	1776	+ // Get the last sitemap URL and clear its transient
	1777	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
	1778	+ if ($sitemap_url) {
	1779	+ delete_transient('mxchat_sitemap_status_' . md5($sitemap_url));
3048	1780	delete_transient('mxchat_last_sitemap_url');
3049	1781	}
3050		-
3051		- if ($pdf_queue_id) {
3052		- // Get PDF path before deleting
3053		- $pdf_path = $this->mxchat_get_queue_meta($pdf_queue_id, 'pdf_path');
3054		-
3055		- $wpdb->delete(
3056		- $table_name,
3057		- array(
3058		- 'queue_id' => $pdf_queue_id,
3059		- 'status' => 'pending'
3060		- ),
3061		- array('%s', '%s')
3062		- );
3063		-
3064		- // Delete PDF file
3065		- if ($pdf_path && file_exists($pdf_path)) {
3066		- wp_delete_file($pdf_path);
3067		- }
3068		-
3069		- delete_transient('mxchat_active_queue_pdf');
	1782	+
	1783	+ // Get the last PDF URL and clear its transient
	1784	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1785	+ if ($pdf_url) {
	1786	+ delete_transient('mxchat_pdf_status_' . md5($pdf_url));
3070	1787	delete_transient('mxchat_last_pdf_url');
3071	1788	}
3072	1789
	1790	+ // Unschedule any pending sitemap events
	1791	+ $timestamp = wp_next_scheduled('mxchat_process_sitemap_urls');
	1792	+ if ($timestamp) {
	1793	+ wp_unschedule_event($timestamp, 'mxchat_process_sitemap_urls');
	1794	+ }
	1795	+
3073	1796	// Redirect back with a success message
3074	1797	set_transient('mxchat_admin_notice_success',
3075	1798	esc_html__('Processing has been stopped successfully.', 'mxchat'),
3076	1799	30
		@@ -3077,12 +1800,8 @@
3077	1800	);
3078	1801	wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
3079	1802	exit;
3080	1803	}
3081		-
3082		-/**
3083		- * Get content list for processing
3084		- */
3085	1804	public function ajax_mxchat_get_content_list() {
3086	1805	// Verify the nonce
3087	1806	check_ajax_referer('mxchat_content_selector_nonce', 'nonce');
3088	1807
		@@ -3090,9 +1809,9 @@
3090	1809	wp_send_json_error(__('Unauthorized access', 'mxchat'));
3091	1810	}
3092	1811
3093	1812	$page = isset($_GET['page']) ? absint($_GET['page']) : 1;
3094		- $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 100;
	1813	+ $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 20;
3095	1814	$search = isset($_GET['search']) ? sanitize_text_field($_GET['search']) : '';
3096	1815	$post_type = isset($_GET['post_type']) ? sanitize_text_field($_GET['post_type']) : 'all';
3097	1816	$post_status = isset($_GET['post_status']) ? sanitize_text_field($_GET['post_status']) : 'publish';
3098	1817	$processed_filter = isset($_GET['processed_filter']) ? sanitize_text_field($_GET['processed_filter']) : 'all';
		@@ -3105,47 +1824,20 @@
3105	1824	'orderby' => 'date',
3106	1825	'order' => 'DESC',
3107	1826	);
3108	1827
3109		- // Handle post types - IMPROVED VERSION
	1828	+ // Handle post types
3110	1829	if ($post_type !== 'all') {
3111	1830	$args['post_type'] = $post_type;
3112	1831	} else {
3113		- // Get all available post types that might contain content
3114		- $all_post_types = array();
	1832	+ // Default to post and page if we can't get post types
	1833	+ $args['post_type'] = array('post', 'page');
3115	1834
3116		- // First get all public post types
3117		- $public_types = get_post_types(array('public' => true), 'names');
3118		- $all_post_types = array_merge($all_post_types, $public_types);
3119		-
3120		- // Add common forum/community post types
3121		- $forum_types = array('topic', 'reply', 'forum', 'wpforo_topic', 'wpforo_post');
3122		- foreach ($forum_types as $forum_type) {
3123		- if (post_type_exists($forum_type)) {
3124		- $all_post_types[] = $forum_type;
3125		- }
	1835	+ // Try to get public post types
	1836	+ $public_types = $this->mxchat_get_public_post_types();
	1837	+ if (is_array($public_types) && !empty($public_types)) {
	1838	+ $args['post_type'] = array_keys($public_types);
3126	1839	}
3127		-
3128		- // Add other commonly used post types
3129		- $common_types = array('product', 'job_listing', 'event', 'portfolio');
3130		- foreach ($common_types as $common_type) {
3131		- if (post_type_exists($common_type)) {
3132		- $all_post_types[] = $common_type;
3133		- }
3134		- }
3135		-
3136		- // Remove duplicates and ensure we have at least some post types
3137		- $all_post_types = array_unique($all_post_types);
3138		-
3139		- if (empty($all_post_types)) {
3140		- // Fallback to basic post types
3141		- $all_post_types = array('post', 'page');
3142		- }
3143		-
3144		- $args['post_type'] = $all_post_types;
3145		-
3146		- // Debug logging to see what post types are being queried
3147		- //error_log('MxChat Debug: Querying post types: ' . implode(', ', $all_post_types));
3148	1840	}
3149	1841
3150	1842	if (!empty($search)) {
3151	1843	$args['s'] = $search;
		@@ -3150,58 +1842,36 @@
3150	1842	if (!empty($search)) {
3151	1843	$args['s'] = $search;
3152	1844	}
3153	1845
3154		- // Get processed data from storage
	1846	+ // ================================
	1847	+ // FIXED: Check only the ACTIVE storage method
	1848	+ // ================================
	1849	+
3155	1850	$processed_data = array();
3156	1851
	1852	+ $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
3157	1853	$pinecone_options = get_option('mxchat_pinecone_addon_options', array());
3158		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3159		-
	1854	+ $pinecone_manager->mxchat_refresh_after_new_content($pinecone_options);
	1855	+
3160	1856	if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3161		- // Get fresh data from Pinecone - no caching
	1857	+ // ONLY check Pinecone if it's enabled
3162	1858	$processed_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
3163	1859	} else {
3164		- // WordPress DB checking with better URL matching for all post types
	1860	+ // ONLY check WordPress DB if Pinecone is not enabled
3165	1861	global $wpdb;
3166	1862	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3167		- $processed_items = $wpdb->get_results("SELECT id, source_url, article_content, timestamp FROM {$table_name}");
3168		-
3169		- // Group items by source_url to count chunks
3170		- $url_chunk_counts = array();
3171		- $url_latest_timestamp = array();
3172		- $url_first_id = array();
3173		-
	1863	+ $processed_items = $wpdb->get_results("SELECT id, source_url, timestamp FROM {$table_name}");
	1864	+
3174	1865	if (!empty($processed_items)) {
3175	1866	foreach ($processed_items as $item) {
3176		- $url = $item->source_url;
3177		- if (empty($url)) continue;
3178		-
3179		- // Count chunks per URL
3180		- if (!isset($url_chunk_counts[$url])) {
3181		- $url_chunk_counts[$url] = 0;
3182		- $url_latest_timestamp[$url] = $item->timestamp;
3183		- $url_first_id[$url] = $item->id;
3184		- }
3185		- $url_chunk_counts[$url]++;
3186		-
3187		- // Track latest timestamp
3188		- if (strtotime($item->timestamp) > strtotime($url_latest_timestamp[$url])) {
3189		- $url_latest_timestamp[$url] = $item->timestamp;
3190		- }
3191		- }
3192		-
3193		- // Now build processed_data with chunk counts
3194		- foreach ($url_chunk_counts as $url => $chunk_count) {
3195		- $post_id = $this->mxchat_url_to_post_id_improved($url);
3196		-
	1867	+ $post_id = url_to_postid($item->source_url);
3197	1868	if ($post_id) {
3198	1869	$processed_data[$post_id] = array(
3199		- 'db_id' => $url_first_id[$url],
3200		- 'timestamp' => $url_latest_timestamp[$url],
3201		- 'url' => $url,
3202		- 'source' => 'wordpress',
3203		- 'chunk_count' => $chunk_count
	1870	+ 'db_id' => $item->id,
	1871	+ 'timestamp' => $item->timestamp,
	1872	+ 'url' => $item->source_url,
	1873	+ 'source' => 'wordpress'
3204	1874	);
3205	1875	}
3206	1876	}
3207	1877	}
		@@ -3206,8 +1876,10 @@
3206	1876	}
3207	1877	}
3208	1878	}
3209	1879
	1880	+ // ================================
	1881	+
3210	1882	// Get processed IDs as a simple array for in_array checks
3211	1883	$processed_ids = array_keys($processed_data);
3212	1884
3213	1885	// Handle processed/unprocessed filter
		@@ -3249,14 +1921,8 @@
3249	1921	$db_record_id = $item_data['db_id'];
3250	1922	}
3251	1923	}
3252	1924
3253		- // Get chunk count for this item
3254		- $chunk_count = 0;
3255		- if ($is_processed && isset($processed_data[$id]['chunk_count'])) {
3256		- $chunk_count = intval($processed_data[$id]['chunk_count']);
3257		- }
3258		-
3259	1925	$content_items[] = array(
3260	1926	'id' => $id,
3261	1927	'title' => get_the_title(),
3262	1928	'permalink' => get_permalink(),
		@@ -3267,10 +1933,9 @@
3267	1933	'word_count' => $word_count,
3268	1934	'already_processed' => $is_processed,
3269	1935	'processed_date' => $processed_date,
3270	1936	'db_record_id' => $db_record_id,
3271		- 'data_source' => $data_source,
3272		- 'chunk_count' => $chunk_count
	1937	+ 'data_source' => $data_source
3273	1938	);
3274	1939	}
3275	1940	wp_reset_postdata();
3276	1941	}
		@@ -3286,178 +1951,8 @@
3286	1951	wp_send_json_success($response);
3287	1952	exit;
3288	1953	}
3289	1954
3290		-
3291		-/**
3292		- * This function handles various WooCommerce URL formats and permalink structures
3293		- */
3294		-private function mxchat_url_to_post_id_improved($url) {
3295		- // First try the standard WordPress function
3296		- $post_id = url_to_postid($url);
3297		-
3298		- if ($post_id > 0) {
3299		- return $post_id;
3300		- }
3301		-
3302		- // If that fails, try more aggressive URL matching
3303		- // Remove trailing slashes and query parameters for better matching
3304		- $clean_url = rtrim($url, '/');
3305		- $clean_url = strtok($clean_url, '?'); // Remove query parameters
3306		-
3307		- // Try again with cleaned URL
3308		- $post_id = url_to_postid($clean_url);
3309		- if ($post_id > 0) {
3310		- return $post_id;
3311		- }
3312		-
3313		- // For bbPress forum topics, try extracting slug from URL
3314		- if (strpos($url, '/topic/') !== false || strpos($url, '/forums/') !== false) {
3315		- // Handle bbPress URLs: /forums/topic/topic-name/
3316		- if (preg_match('/\/forums\/topic\/([^\/\?]+)/', $url, $matches)) {
3317		- $topic_slug = $matches[1];
3318		-
3319		- // Look up topic by slug
3320		- $topic = get_page_by_path($topic_slug, OBJECT, 'topic');
3321		- if ($topic) {
3322		- return $topic->ID;
3323		- }
3324		-
3325		- // Alternative method: query by post_name
3326		- global $wpdb;
3327		- $post_id = $wpdb->get_var($wpdb->prepare(
3328		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3329		- $topic_slug
3330		- ));
3331		-
3332		- if ($post_id) {
3333		- return intval($post_id);
3334		- }
3335		- }
3336		-
3337		- // Handle simpler topic URLs: /topic/topic-name/
3338		- if (preg_match('/\/topic\/([^\/\?]+)/', $url, $matches)) {
3339		- $topic_slug = $matches[1];
3340		-
3341		- global $wpdb;
3342		- $post_id = $wpdb->get_var($wpdb->prepare(
3343		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3344		- $topic_slug
3345		- ));
3346		-
3347		- if ($post_id) {
3348		- return intval($post_id);
3349		- }
3350		- }
3351		- }
3352		-
3353		- // For WooCommerce products
3354		- if (strpos($url, '/product/') !== false || strpos($url, 'product=') !== false) {
3355		- // Extract product slug from various URL formats
3356		- $product_slug = '';
3357		-
3358		- // Handle pretty permalinks: /product/product-name/
3359		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
3360		- $product_slug = $matches[1];
3361		- }
3362		- // Handle query parameters: ?product=product-name
3363		- elseif (preg_match('/[\?&]product=([^&]+)/', $url, $matches)) {
3364		- $product_slug = $matches[1];
3365		- }
3366		-
3367		- if (!empty($product_slug)) {
3368		- // Look up product by slug
3369		- $product = get_page_by_path($product_slug, OBJECT, 'product');
3370		- if ($product) {
3371		- return $product->ID;
3372		- }
3373		-
3374		- // Alternative method: query by post_name
3375		- global $wpdb;
3376		- $post_id = $wpdb->get_var($wpdb->prepare(
3377		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'product' AND post_status = 'publish'",
3378		- $product_slug
3379		- ));
3380		-
3381		- if ($post_id) {
3382		- return intval($post_id);
3383		- }
3384		- }
3385		- }
3386		-
3387		- // Generic approach: try to extract slug and match against all post types
3388		- $parsed_url = wp_parse_url($clean_url);
3389		- $path = $parsed_url['path'] ?? '';
3390		-
3391		- if (!empty($path)) {
3392		- // Get the last part of the path as potential slug
3393		- $path_parts = array_filter(explode('/', trim($path, '/')));
3394		- $potential_slug = end($path_parts);
3395		-
3396		- if (!empty($potential_slug)) {
3397		- global $wpdb;
3398		-
3399		- // Try to find any post with this slug
3400		- $post_id = $wpdb->get_var($wpdb->prepare(
3401		- "SELECT ID FROM {$wpdb->posts}
3402		- WHERE post_name = %s
3403		- AND post_status IN ('publish', 'closed', 'private')
3404		- AND post_type NOT IN ('revision', 'attachment', 'nav_menu_item')
3405		- ORDER BY CASE
3406		- WHEN post_type = 'post' THEN 1
3407		- WHEN post_type = 'page' THEN 2
3408		- WHEN post_type = 'topic' THEN 3
3409		- WHEN post_type = 'product' THEN 4
3410		- ELSE 5
3411		- END
3412		- LIMIT 1",
3413		- $potential_slug
3414		- ));
3415		-
3416		- if ($post_id) {
3417		- return intval($post_id);
3418		- }
3419		- }
3420		- }
3421		-
3422		- // ADDITIONAL: Try direct database lookup by URL variations
3423		- global $wpdb;
3424		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3425		-
3426		- // Try variations of the URL (with/without trailing slash, http/https)
3427		- $url_variations = array(
3428		- $url,
3429		- rtrim($url, '/'),
3430		- $url . '/',
3431		- str_replace('http://', 'https://', $url),
3432		- str_replace('https://', 'http://', $url),
3433		- str_replace('http://', 'https://', rtrim($url, '/')),
3434		- str_replace('https://', 'http://', rtrim($url, '/'))
3435		- );
3436		-
3437		- // Remove duplicates
3438		- $url_variations = array_unique($url_variations);
3439		-
3440		- foreach ($url_variations as $variation) {
3441		- $existing_record = $wpdb->get_row($wpdb->prepare(
3442		- "SELECT id, source_url FROM $table_name WHERE source_url = %s",
3443		- $variation
3444		- ));
3445		-
3446		- if ($existing_record) {
3447		- // Try to get post ID from this stored URL
3448		- $stored_post_id = url_to_postid($existing_record->source_url);
3449		- if ($stored_post_id > 0) {
3450		- return $stored_post_id;
3451		- }
3452		- }
3453		- }
3454		-
3455		- return 0; // No match found
3456		-}
3457		-/**
3458		- * Process selected content via AJAX
3459		- */
3460	1955	public function ajax_mxchat_process_selected_content() {
3461	1956	// Basic request validation
3462	1957	if (!check_ajax_referer('mxchat_content_selector_nonce', 'nonce', false)) {
3463	1958	wp_send_json_error('Invalid nonce');
		@@ -3481,11 +1976,8 @@
3481	1976	wp_send_json_error('No content selected');
3482	1977	exit;
3483	1978	}
3484	1979
3485		- // Get bot_id from request
3486		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
3487		-
3488	1980	// Process only ONE post at a time to avoid request size issues
3489	1981	$post_id = reset($post_ids);
3490	1982	$post = get_post($post_id);
3491	1983
		@@ -3492,151 +1984,15 @@
3492	1984	if (!$post) {
3493	1985	wp_send_json_error('Post not found');
3494	1986	exit;
3495	1987	}
3496		-
3497		- // Allow developers to modify post data before processing into knowledge base
3498		- $post = apply_filters('mxchat_before_process_post', $post, $bot_id);
3499		-
3500		- // Get content including title, short description (for WooCommerce), and main content
3501		- $content = $post->post_title . "\n\n";
3502		-
3503		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
3504		- if (!empty($post->post_excerpt)) {
3505		- // Remove shortcode tags but preserve content inside them
3506		- $clean_excerpt = $this->strip_shortcode_tags_preserve_content($post->post_excerpt);
3507		- $content .= "Short Description: " . wp_strip_all_tags($clean_excerpt) . "\n\n";
3508		- }
3509		-
3510		- // Add main content - remove shortcode tags but preserve content inside them
3511		- $clean_content = $this->strip_shortcode_tags_preserve_content($post->post_content);
3512		- $content .= wp_strip_all_tags($clean_content);
3513		-
3514		- // ADD WOOCOMMERCE PRODUCT DATA (pricing, stock, categories, custom tabs)
3515		- if (get_post_type($post_id) === 'product' && class_exists('WooCommerce')) {
3516		- $product = wc_get_product($post_id);
3517		-
3518		- if ($product) {
3519		- // Get pricing information
3520		- $regular_price = $product->get_regular_price();
3521		- $sale_price = $product->get_sale_price();
3522		- $price = $product->get_price();
3523		- $sku = $product->get_sku();
3524		-
3525		- // Get currency symbol
3526		- $currency_symbol = get_woocommerce_currency_symbol();
3527		-
3528		- // Add pricing information
3529		- $content .= "\n";
3530		- if (!empty($regular_price)) {
3531		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
3532		- } elseif (!empty($price)) {
3533		- $content .= "Price: " . $currency_symbol . $price . "\n";
3534		- }
3535		-
3536		- if (!empty($sale_price) && $sale_price !== $regular_price) {
3537		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
3538		- }
3539		-
3540		- // Handle variable products - show price range
3541		- if ($product->is_type('variable')) {
3542		- $min_price = $product->get_variation_price('min');
3543		- $max_price = $product->get_variation_price('max');
3544		- if ($min_price !== $max_price) {
3545		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
3546		- }
3547		- }
3548		-
3549		- if (!empty($sku)) {
3550		- $content .= "SKU: " . $sku . "\n";
3551		- }
3552		-
3553		- // Get product categories
3554		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
3555		- if (!empty($categories) && !is_wp_error($categories)) {
3556		- $content .= "Categories: " . implode(', ', $categories) . "\n";
3557		- }
3558		- }
3559		-
3560		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
3561		- $custom_tabs = get_post_meta($post_id, 'yikes_woo_products_tabs', true);
3562		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
3563		- foreach ($custom_tabs as $tab) {
3564		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
3565		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
3566		-
3567		- if (!empty($tab_title) && !empty($tab_content)) {
3568		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
3569		- }
3570		- }
3571		- }
3572		-
3573		- // Also check for reusable/saved tabs applied to this product
3574		- $applied_saved_tabs = get_post_meta($post_id, 'yikes_woo_reusable_products_tabs_applied', true);
3575		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
3576		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
3577		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
3578		- foreach ($applied_saved_tabs as $saved_tab_id) {
3579		- if (isset($saved_tabs[$saved_tab_id])) {
3580		- $tab = $saved_tabs[$saved_tab_id];
3581		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
3582		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
3583		-
3584		- if (!empty($tab_title) && !empty($tab_content)) {
3585		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
3586		- }
3587		- }
3588		- }
3589		- }
3590		- }
3591		- }
3592		-
3593		- // ADD ACF FIELDS SUPPORT
3594		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
3595		- if (!empty($acf_fields)) {
3596		- $acf_content_parts = array();
3597		-
3598		- foreach ($acf_fields as $field_name => $field_value) {
3599		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
3600		-
3601		- if (!empty($formatted_value)) {
3602		- $field_label = ucwords(str_replace('_', ' ', $field_name));
3603		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
3604		- }
3605		- }
3606		-
3607		- if (!empty($acf_content_parts)) {
3608		- $content .= "\n\n" . implode("\n", $acf_content_parts);
3609		- }
3610		- }
3611		-
3612		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
3613		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
3614		- if (!empty($custom_meta)) {
3615		- $meta_content_parts = array();
3616		-
3617		- foreach ($custom_meta as $meta_key => $meta_value) {
3618		- // Convert meta key to readable label
3619		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
3620		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
3621		- }
3622		-
3623		- if (!empty($meta_content_parts)) {
3624		- $content .= "\n\n" . implode("\n", $meta_content_parts);
3625		- }
3626		- }
3627		-
3628		- // Debug logging for WordPress Import content
3629		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Post ID: ' . $post_id . ' Title: ' . $post->post_title);
3630		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Raw post_content length: ' . strlen($post->post_content));
3631		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Final content length: ' . strlen($content));
3632		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Content preview: ' . substr($content, 0, 300));
3633		-
3634		- // Note: Removed 10,000 char limit - chunking now handles large content properly
3635		-
3636		- // Get bot-specific API key
3637		- $bot_options = $this->get_bot_options($bot_id);
3638		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	1988	+
	1989	+ // Get minimal content
	1990	+ $content = $post->post_title . "\n\n" . wp_strip_all_tags($post->post_content);
	1991	+ $content = substr($content, 0, 10000); // Limit content size
	1992	+
	1993	+ // Get API key with proper model detection
	1994	+ $options = get_option('mxchat_options');
3639	1995	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
3640	1996
3641	1997	if (strpos($selected_model, 'voyage') === 0) {
3642	1998	$api_key = $options['voyage_api_key'] ?? '';
		@@ -3649,9 +2005,8 @@
3649	2005	$provider_name = 'OpenAI';
3650	2006	}
3651	2007
3652	2008	if (empty($api_key)) {
3653		- MxChat_Admin::mxchat_log_debug('api_error', $provider_name . ' API key not configured for knowledge processing');
3654	2009	wp_send_json_error($provider_name . ' API key not configured');
3655	2010	exit;
3656	2011	}
3657	2012
		@@ -3657,23 +2012,26 @@
3657	2012
3658	2013	$source_url = get_permalink($post_id);
3659	2014	$vector_id = md5($source_url); // Vector ID for Pinecone
3660	2015
3661		- // Check for existing content in bot-specific storage
	2016	+ // ================================
	2017	+ // FIXED: Check for existing content in ONLY the active storage method
	2018	+ // ================================
	2019	+
3662	2020	$is_update = false;
3663	2021
3664		- // Get bot-specific Pinecone configuration
3665		- $bot_pinecone_config = $this->get_bot_pinecone_config($bot_id);
3666		- $use_pinecone = !empty($bot_pinecone_config) && ($bot_pinecone_config['use_pinecone'] ?? false);
	2022	+ // Check if Pinecone is enabled
	2023	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	2024	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3667	2025
3668		- if ($use_pinecone && !empty($bot_pinecone_config['api_key'])) {
3669		- // Check Pinecone for this bot
3670		- $pinecone_data = $this->mxchat_get_pinecone_processed_content($bot_pinecone_config);
	2026	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2027	+ // ONLY check Pinecone if it's enabled
	2028	+ $pinecone_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
3671	2029	if (isset($pinecone_data[$post_id])) {
3672	2030	$is_update = true;
3673	2031	}
3674	2032	} else {
3675		- // Check WordPress DB (same as before since it's shared)
	2033	+ // ONLY check WordPress DB if Pinecone is not enabled
3676	2034	global $wpdb;
3677	2035	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3678	2036	$existing_record = $wpdb->get_row($wpdb->prepare(
3679	2037	"SELECT id FROM $table_name WHERE source_url = %s",
		@@ -3683,54 +2041,54 @@
3683	2041	if ($existing_record) {
3684	2042	$is_update = true;
3685	2043	}
3686	2044	}
3687		-
3688		- // UPDATED 2.5.6: Determine content type based on post_type
3689		- $post_type = $post->post_type;
3690		- $content_type = 'content'; // Default fallback
3691		-
3692		- // Map WordPress post types to content types
3693		- switch ($post_type) {
3694		- case 'post':
3695		- $content_type = 'post';
3696		- break;
3697		- case 'page':
3698		- $content_type = 'page';
3699		- break;
3700		- case 'product':
3701		- $content_type = 'product';
3702		- break;
3703		- default:
3704		- // For custom post types, use the post type name
3705		- $content_type = sanitize_key($post_type);
3706		- break;
3707		- }
3708		-
3709		- // Use the centralized utility function with bot_id and content_type
	2045	+
	2046	+ // Use the centralized utility function for storage
3710	2047	$result = MxChat_Utils::submit_content_to_db(
3711		- $content,
3712		- $source_url,
	2048	+ $content,
	2049	+ $source_url,
3713	2050	$api_key,
3714		- $vector_id,
3715		- $bot_id,
3716		- $content_type
	2051	+ $vector_id
3717	2052	);
3718		-
	2053	+
3719	2054	if (is_wp_error($result)) {
3720		- MxChat_Admin::mxchat_log_debug('storage_error', 'Knowledge storage failed: ' . $result->get_error_message(), array('source_url' => $source_url));
3721	2055	wp_send_json_error('Storage failed: ' . $result->get_error_message());
3722	2056	exit;
3723	2057	}
3724		-
3725		- // Automatically apply role restriction based on tags
3726		- $this->apply_role_restriction_to_post($post_id, $source_url);
3727		-
	2058	+
	2059	+ // ================================
	2060	+ // UPDATE: Only update caches if Pinecone is enabled
	2061	+ // ================================
	2062	+
	2063	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2064	+ // Update vector ID cache for improved fetching
	2065	+ $this->mxchat_update_pinecone_vector_cache($vector_id);
	2066	+
	2067	+ // Update local processed content cache for immediate UI feedback
	2068	+ $pinecone_cache = get_option('mxchat_pinecone_processed_cache', array());
	2069	+ $pinecone_cache[$post_id] = array(
	2070	+ 'db_id' => $vector_id,
	2071	+ 'processed_date' => 'Just now',
	2072	+ 'url' => $source_url,
	2073	+ 'source' => 'pinecone',
	2074	+ 'timestamp' => current_time('timestamp')
	2075	+ );
	2076	+ update_option('mxchat_pinecone_processed_cache', $pinecone_cache);
	2077	+
	2078	+ // Also update the general processed content cache
	2079	+ $processed_cache = get_option('mxchat_processed_content_cache', array());
	2080	+ $processed_cache[$post_id] = array(
	2081	+ 'db_id' => $vector_id,
	2082	+ 'timestamp' => current_time('timestamp'),
	2083	+ 'url' => $source_url,
	2084	+ 'source' => 'pinecone'
	2085	+ );
	2086	+ update_option('mxchat_processed_content_cache', $processed_cache);
	2087	+ }
	2088	+
3728	2089	$operation_type = $is_update ? 'update' : 'new';
3729	2090
3730		- // Count ACF fields for debugging
3731		- $acf_field_count = count($acf_fields);
3732		-
3733	2091	// Success response with minimal data
3734	2092	wp_send_json_success(array(
3735	2093	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
3736	2094	'post_id' => $post_id,
		@@ -3735,94 +2093,27 @@
3735	2093	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
3736	2094	'post_id' => $post_id,
3737	2095	'title' => $post->post_title,
3738	2096	'operation_type' => $operation_type,
3739		- 'vector_id' => $vector_id,
3740		- 'acf_fields_found' => $acf_field_count,
3741		- 'content_preview' => substr($content, 0, 100) . '...',
3742		- 'bot_id' => $bot_id
	2097	+ 'vector_id' => $vector_id, // Include vector ID for debugging
	2098	+ 'cache_updated' => $use_pinecone // Indicate if cache was updated
3743	2099	));
3744	2100	exit;
3745	2101	}
3746	2102
3747		-private function apply_role_restriction_to_post($post_id, $source_url) {
3748		- // Get tag-role mappings
3749		- $mappings = get_option('mxchat_tag_role_mappings', array());
3750		-
3751		- if (empty($mappings)) {
3752		- return; // No mappings, leave as public
3753		- }
3754		-
3755		- // Get all tags for the post
3756		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
3757		-
3758		- if (empty($post_tags)) {
3759		- return; // No tags, leave as public
3760		- }
3761		-
3762		- // Determine the highest role restriction based on tags
3763		- $highest_role = 'public';
3764		- $role_hierarchy = array(
3765		- 'public' => 0,
3766		- 'logged_in' => 1,
3767		- 'subscriber' => 2,
3768		- 'contributor' => 3,
3769		- 'author' => 4,
3770		- 'editor' => 5,
3771		- 'administrator' => 6
3772		- );
3773		-
3774		- foreach ($post_tags as $tag_slug) {
3775		- if (isset($mappings[$tag_slug])) {
3776		- $role = $mappings[$tag_slug];
3777		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
3778		- $highest_role = $role;
3779		- }
3780		- }
3781		- }
3782		-
3783		- // If no restricted tags found, return (leave as public)
3784		- if ($highest_role === 'public') {
3785		- return;
3786		- }
3787		-
3788		- // Update the role restriction in the database
3789		- global $wpdb;
3790		-
3791		- // Check if using Pinecone
3792		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
3793		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3794		-
3795		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3796		- // Update Pinecone role restriction
3797		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
3798		- $vector_id = md5($source_url);
3799		-
3800		- $wpdb->replace(
3801		- $roles_table,
3802		- array(
3803		- 'vector_id' => $vector_id,
3804		- 'role_restriction' => $highest_role,
3805		- 'updated_at' => current_time('mysql')
3806		- ),
3807		- array('%s', '%s', '%s')
3808		- );
3809		- } else {
3810		- // Update WordPress DB
3811		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3812		-
3813		- $wpdb->update(
3814		- $table_name,
3815		- array('role_restriction' => $highest_role),
3816		- array('source_url' => $source_url),
3817		- array('%s'),
3818		- array('%s')
3819		- );
3820		- }
3821		-}
3822	2103
	2104	+
	2105	+ /**
	2106	+ * Updates cache with new vector ID if absent
	2107	+ */
	2108	+ public function mxchat_update_pinecone_vector_cache($vector_id) {
	2109	+ $cached_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2110	+ if (!in_array($vector_id, $cached_ids)) {
	2111	+ $cached_ids[] = $vector_id;
	2112	+ update_option('mxchat_pinecone_vector_ids_cache', $cached_ids);
	2113	+ }
	2114	+ }
3823	2115	public function mxchat_get_public_post_types() {
3824		- // Get all public post types
3825	2116	$post_types = get_post_types(array('public' => true), 'objects');
3826	2117	$post_type_options = array();
3827	2118
3828	2119	foreach ($post_types as $post_type) {
		@@ -3828,45 +2119,51 @@
3828	2119	foreach ($post_types as $post_type) {
3829	2120	$post_type_options[$post_type->name] = $post_type->label;
3830	2121	}
3831	2122
3832		- // Also include common forum/community post types that might not be marked as public
3833		- $additional_types = array(
3834		- 'topic' => 'Forum Topics (bbPress)',
3835		- 'reply' => 'Forum Replies (bbPress)',
3836		- 'forum' => 'Forums (bbPress)',
3837		- 'wpforo_topic' => 'wpForo Topics',
3838		- 'wpforo_post' => 'wpForo Posts'
3839		- );
3840		-
3841		- foreach ($additional_types as $type_name => $type_label) {
3842		- if (post_type_exists($type_name) && !isset($post_type_options[$type_name])) {
3843		- $post_type_options[$type_name] = $type_label;
3844		- }
3845		- }
3846		-
3847	2123	return $post_type_options;
3848	2124	}
	2125	+public function mxchat_get_pinecone_processed_content($pinecone_options) {
	2126	+ //error_log('=== DEBUG: Starting mxchat_get_pinecone_processed_content ===');
	2127	+
	2128	+ // First check local cache for immediate updates
	2129	+ $cached_data = get_option('mxchat_pinecone_processed_cache', array());
	2130	+ //error_log('DEBUG: Found ' . count($cached_data) . ' items in local cache');
3849	2131
3850		-/**
3851		- * Retrieves processed content from Pinecone API
3852		- */
3853		-public function mxchat_get_pinecone_processed_content($pinecone_options) {
3854	2132	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
3855	2133	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
	2134	+
	2135	+ //error_log('DEBUG: API key present: ' . (!empty($api_key) ? 'YES' : 'NO'));
	2136	+ //error_log('DEBUG: Host: ' . $host);
3856	2137
3857	2138	if (empty($api_key) || empty($host)) {
3858		- return array();
	2139	+ //error_log('DEBUG: Missing API credentials, returning cached data only');
	2140	+ return $cached_data;
3859	2141	}
3860	2142
3861	2143	$pinecone_data = array();
3862	2144
3863	2145	try {
3864		- // Always get fresh data from Pinecone
3865		- $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
	2146	+ // Method 1: Try to get vectors using cached vector IDs first
	2147	+ $cached_vector_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2148	+ //error_log('DEBUG: Found ' . count($cached_vector_ids) . ' cached vector IDs');
3866	2149
3867		- // Method 2: Final fallback - try stats endpoint (if available)
	2150	+ if (!empty($cached_vector_ids)) {
	2151	+ //error_log('DEBUG: Trying to fetch by cached vector IDs...');
	2152	+ $pinecone_data = $this->mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $cached_vector_ids);
	2153	+ //error_log('DEBUG: Fetch by IDs returned ' . count($pinecone_data) . ' items');
	2154	+ }
	2155	+
	2156	+ // Method 2: If no cached IDs or fetch failed, use scanning approach
3868	2157	if (empty($pinecone_data)) {
	2158	+ //error_log('DEBUG: Trying scanning approach...');
	2159	+ $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
	2160	+ //error_log('DEBUG: Scanning returned ' . count($pinecone_data) . ' items');
	2161	+ }
	2162	+
	2163	+ // Method 3: Final fallback - try stats endpoint
	2164	+ if (empty($pinecone_data)) {
	2165	+ //error_log('DEBUG: Trying stats endpoint...');
3869	2166	$stats_url = "https://{$host}/describe_index_stats";
3870	2167
3871	2168	$response = wp_remote_post($stats_url, array(
3872	2169	'headers' => array(
		@@ -3879,17 +2176,44 @@
3879	2176
3880	2177	if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3881	2178	$body = wp_remote_retrieve_body($response);
3882	2179	$stats_data = json_decode($body, true);
	2180	+ //error_log('DEBUG: Pinecone stats: ' . print_r($stats_data, true));
	2181	+ } else {
	2182	+ if (is_wp_error($response)) {
	2183	+ //error_log('DEBUG: Stats endpoint error: ' . $response->get_error_message());
	2184	+ } else {
	2185	+ //error_log('DEBUG: Stats endpoint failed with code: ' . wp_remote_retrieve_response_code($response));
	2186	+ }
3883	2187	}
3884	2188	}
3885	2189
3886	2190	} catch (Exception $e) {
3887		- // Log error but return fresh data only
	2191	+ //error_log('DEBUG: Exception in get_pinecone_processed_content: ' . $e->getMessage());
3888	2192	}
3889	2193
3890		- return $pinecone_data;
	2194	+ // Merge cached data with Pinecone data
	2195	+ $merged_data = $pinecone_data;
	2196	+
	2197	+ foreach ($cached_data as $post_id => $cache_item) {
	2198	+ $cache_timestamp = $cache_item['timestamp'] ?? 0;
	2199	+ $time_diff = current_time('timestamp') - $cache_timestamp;
	2200	+
	2201	+ if ($time_diff < 300) { // 5 minutes = 300 seconds
	2202	+ $merged_data[$post_id] = $cache_item;
	2203	+ } else {
	2204	+ if (!isset($merged_data[$post_id])) {
	2205	+ $merged_data[$post_id] = $cache_item;
	2206	+ }
	2207	+ }
	2208	+ }
	2209	+
	2210	+ //error_log('DEBUG: Final merged data count: ' . count($merged_data));
	2211	+ //error_log('=== DEBUG: End mxchat_get_pinecone_processed_content ===');
	2212	+
	2213	+ return $merged_data;
3891	2214	}
	2215	+
3892	2216	public function mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $vector_ids) {
3893	2217	//error_log('=== DEBUG: Starting mxchat_fetch_pinecone_vectors_by_ids ===');
3894	2218
3895	2219	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
		@@ -3981,46 +2305,16 @@
3981	2305	return array();
3982	2306	}
3983	2307	}
3984	2308
3985		-/**
3986		- * Get embedding dimensions based on the selected model.
3987		- */
3988		-private function mxchat_get_embedding_dimensions() {
3989		- $options = get_option('mxchat_options', array());
3990		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
3991		-
3992		- $model_dimensions = array(
3993		- 'text-embedding-ada-002' => 1536,
3994		- 'text-embedding-3-small' => 1536,
3995		- 'text-embedding-3-large' => 3072,
3996		- 'voyage-2' => 1024,
3997		- 'voyage-large-2' => 1536,
3998		- 'voyage-3-large' => 2048,
3999		- 'gemini-embedding-001' => 1536,
4000		- );
4001		-
4002		- if (strpos($selected_model, 'voyage-3-large') === 0) {
4003		- $custom_dimensions = $options['voyage_output_dimension'] ?? 2048;
4004		- return intval($custom_dimensions);
4005		- }
4006		-
4007		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4008		- $custom_dimensions = $options['gemini_output_dimension'] ?? 1536;
4009		- return intval($custom_dimensions);
4010		- }
4011		-
4012		- return $model_dimensions[$selected_model] ?? 1536;
4013		-}
4014		-
4015		-/**
4016		- * Scan Pinecone for processed content
4017		- */
4018	2309	public function mxchat_scan_pinecone_for_processed_content($pinecone_options) {
	2310	+ //error_log('=== DEBUG: Starting mxchat_scan_pinecone_for_processed_content ===');
	2311	+
4019	2312	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4020	2313	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
4021	2314
4022	2315	if (empty($api_key) || empty($host)) {
	2316	+ //error_log('DEBUG: Missing API credentials for scanning');
4023	2317	return array();
4024	2318	}
4025	2319
4026	2320	try {
		@@ -4027,18 +2321,17 @@
4027	2321	// Use multiple random vectors to get better coverage
4028	2322	$all_matches = array();
4029	2323	$seen_ids = array();
4030	2324
4031		- // Get correct dimensions for the configured embedding model
4032		- $dimensions = $this->mxchat_get_embedding_dimensions();
4033		-
4034	2325	// Try 3 different random vectors to get better coverage
4035	2326	for ($i = 0; $i < 3; $i++) {
	2327	+ //error_log('DEBUG: Scanning attempt ' . ($i + 1) . '/3');
	2328	+
4036	2329	$query_url = "https://{$host}/query";
4037	2330
4038	2331	// Generate a random unit vector instead of zeros
4039	2332	$random_vector = array();
4040		- for ($j = 0; $j < $dimensions; $j++) {
	2333	+ for ($j = 0; $j < 1536; $j++) {
4041	2334	$random_vector[] = (rand(-1000, 1000) / 1000.0);
4042	2335	}
4043	2336
4044	2337	// Normalize the vector to unit length
		@@ -4063,14 +2356,18 @@
4063	2356	'timeout' => 30
4064	2357	));
4065	2358
4066	2359	if (is_wp_error($response)) {
	2360	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' WP error: ' . $response->get_error_message());
4067	2361	continue;
4068	2362	}
4069	2363
4070	2364	$response_code = wp_remote_retrieve_response_code($response);
	2365	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' response code: ' . $response_code);
4071	2366
4072	2367	if ($response_code !== 200) {
	2368	+ $error_body = wp_remote_retrieve_body($response);
	2369	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' failed with body: ' . substr($error_body, 0, 500));
4073	2370	continue;
4074	2371	}
4075	2372
4076	2373	$body = wp_remote_retrieve_body($response);
		@@ -4076,8 +2373,9 @@
4076	2373	$body = wp_remote_retrieve_body($response);
4077	2374	$data = json_decode($body, true);
4078	2375
4079	2376	if (isset($data['matches'])) {
	2377	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' returned ' . count($data['matches']) . ' matches');
4080	2378	foreach ($data['matches'] as $match) {
4081	2379	$match_id = $match['id'] ?? '';
4082	2380	if (!empty($match_id) && !isset($seen_ids[$match_id])) {
4083	2381	$all_matches[] = $match;
		@@ -4083,14 +2381,18 @@
4083	2381	$all_matches[] = $match;
4084	2382	$seen_ids[$match_id] = true;
4085	2383	}
4086	2384	}
	2385	+ } else {
	2386	+ //error_log('DEBUG: Query attempt ' . ($i + 1) . ' - no matches key in response');
4087	2387	}
4088	2388	}
4089	2389
4090		- // Convert matches to processed data format, grouping by URL to count chunks
	2390	+ //error_log('DEBUG: Total unique matches found: ' . count($all_matches));
	2391	+
	2392	+ // Convert matches to processed data format
4091	2393	$processed_data = array();
4092		- $url_chunk_counts = array();
	2394	+ $vector_ids_for_cache = array();
4093	2395
4094	2396	foreach ($all_matches as $match) {
4095	2397	$metadata = $match['metadata'] ?? array();
4096	2398	$source_url = $metadata['source_url'] ?? '';
		@@ -4098,14 +2400,8 @@
4098	2400
4099	2401	if (!empty($source_url) && !empty($match_id)) {
4100	2402	$post_id = url_to_postid($source_url);
4101	2403	if ($post_id) {
4102		- // Count chunks per post_id
4103		- if (!isset($url_chunk_counts[$post_id])) {
4104		- $url_chunk_counts[$post_id] = 0;
4105		- }
4106		- $url_chunk_counts[$post_id]++;
4107		-
4108	2404	$created_at = $metadata['created_at'] ?? '';
4109	2405	$processed_date = 'Recently';
4110	2406
4111	2407	if (!empty($created_at)) {
		@@ -4114,311 +2410,243 @@
4114	2410	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
4115	2411	}
4116	2412	}
4117	2413
4118		- // Only store if not already set, or update with newer timestamp
4119		- if (!isset($processed_data[$post_id]) ||
4120		- ($timestamp ?? 0) > ($processed_data[$post_id]['timestamp'] ?? 0)) {
4121		- $processed_data[$post_id] = array(
4122		- 'db_id' => $match_id,
4123		- 'processed_date' => $processed_date,
4124		- 'url' => $source_url,
4125		- 'source' => 'pinecone',
4126		- 'timestamp' => $timestamp ?? current_time('timestamp')
4127		- );
4128		- }
	2414	+ $processed_data[$post_id] = array(
	2415	+ 'db_id' => $match_id,
	2416	+ 'processed_date' => $processed_date,
	2417	+ 'url' => $source_url,
	2418	+ 'source' => 'pinecone',
	2419	+ 'timestamp' => $timestamp ?? current_time('timestamp')
	2420	+ );
	2421	+
	2422	+ $vector_ids_for_cache[] = $match_id;
4129	2423	}
4130	2424	}
4131	2425	}
4132	2426
4133		- // Add chunk counts to processed data
4134		- foreach ($url_chunk_counts as $post_id => $chunk_count) {
4135		- if (isset($processed_data[$post_id])) {
4136		- $processed_data[$post_id]['chunk_count'] = $chunk_count;
4137		- }
	2427	+ // Update the vector IDs cache for future use
	2428	+ if (!empty($vector_ids_for_cache)) {
	2429	+ update_option('mxchat_pinecone_vector_ids_cache', $vector_ids_for_cache);
	2430	+ //error_log('DEBUG: Updated vector IDs cache with ' . count($vector_ids_for_cache) . ' IDs');
4138	2431	}
4139	2432
	2433	+ //error_log('DEBUG: Returning ' . count($processed_data) . ' processed items from scanning');
4140	2434	return $processed_data;
4141	2435
4142	2436	} catch (Exception $e) {
	2437	+ //error_log('DEBUG: Exception in scan_pinecone_for_processed_content: ' . $e->getMessage());
4143	2438	return array();
4144	2439	}
4145	2440	}
4146		-/**
4147		- * Generate embeddings from input text for MXChat with bot support
4148		- */
4149		-private function mxchat_generate_embedding($text, $bot_id = 'default') {
4150		- // Enable detailed logging for debugging
4151		- //error_log('[MXCHAT-EMBED] Starting embedding generation for bot: ' . $bot_id . '. Text length: ' . strlen($text) . ' bytes');
4152		- //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4153	2441
4154		- // Get bot-specific options
4155		- $bot_options = $this->get_bot_options($bot_id);
4156		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
4157		-
4158		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4159		- //error_log('[MXCHAT-EMBED] Selected embedding model for bot ' . $bot_id . ': ' . $selected_model);
	2442	+ /**
	2443	+ * Generates embeddings from input text for MXChat
	2444	+ */
	2445	+ private function mxchat_generate_embedding($text) {
	2446	+ // Enable detailed logging for debugging
	2447	+ //error_log('[MXCHAT-EMBED] Starting embedding generation. Text length: ' . strlen($text) . ' bytes');
	2448	+ //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4160	2449
4161		- // Determine provider and endpoint
4162		- if (strpos($selected_model, 'voyage') === 0) {
4163		- $api_key = $options['voyage_api_key'] ?? '';
4164		- $endpoint = 'https://api.voyageai.com/v1/embeddings';
4165		- $provider_name = 'Voyage AI';
4166		- //error_log('[MXCHAT-EMBED] Using Voyage AI API for bot ' . $bot_id);
4167		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
4168		- $api_key = $options['gemini_api_key'] ?? '';
4169		- $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
4170		- $provider_name = 'Google Gemini';
4171		- //error_log('[MXCHAT-EMBED] Using Google Gemini API for bot ' . $bot_id);
4172		- } else {
4173		- $api_key = $options['api_key'] ?? '';
4174		- $endpoint = 'https://api.openai.com/v1/embeddings';
4175		- $provider_name = 'OpenAI';
4176		- //error_log('[MXCHAT-EMBED] Using OpenAI API for bot ' . $bot_id);
4177		- }
	2450	+ $options = get_option('mxchat_options');
	2451	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	2452	+ //error_log('[MXCHAT-EMBED] Selected embedding model: ' . $selected_model);
4178	2453
4179		- //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
	2454	+ // Determine provider and endpoint
	2455	+ if (strpos($selected_model, 'voyage') === 0) {
	2456	+ $api_key = $options['voyage_api_key'] ?? '';
	2457	+ $endpoint = 'https://api.voyageai.com/v1/embeddings';
	2458	+ $provider_name = 'Voyage AI';
	2459	+ //error_log('[MXCHAT-EMBED] Using Voyage AI API');
	2460	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	2461	+ $api_key = $options['gemini_api_key'] ?? '';
	2462	+ $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
	2463	+ $provider_name = 'Google Gemini';
	2464	+ //error_log('[MXCHAT-EMBED] Using Google Gemini API');
	2465	+ } else {
	2466	+ $api_key = $options['api_key'] ?? '';
	2467	+ $endpoint = 'https://api.openai.com/v1/embeddings';
	2468	+ $provider_name = 'OpenAI';
	2469	+ //error_log('[MXCHAT-EMBED] Using OpenAI API');
	2470	+ }
4180	2471
4181		- if (empty($api_key)) {
4182		- $error_message = sprintf('Missing %s API key for bot %s. Please configure your API key in the bot settings.', $provider_name, $bot_id);
4183		- //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
4184		- return $error_message;
4185		- }
	2472	+ //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
4186	2473
4187		- // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
4188		- $estimated_tokens = ceil(str_word_count($text) / 0.75);
4189		- //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
	2474	+ if (empty($api_key)) {
	2475	+ $error_message = sprintf('Missing %s API key. Please configure your API key in the MxChat settings.', $provider_name);
	2476	+ //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
	2477	+ return $error_message;
	2478	+ }
4190	2479
4191		- if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
4192		- //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
4193		- // Consider truncating text here
4194		- }
	2480	+ // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
	2481	+ $estimated_tokens = ceil(str_word_count($text) / 0.75);
	2482	+ //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
4195	2483
4196		- // Prepare request body based on provider
4197		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4198		- // Gemini API format
4199		- $request_body = array(
4200		- 'model' => 'models/' . $selected_model,
4201		- 'content' => array(
4202		- 'parts' => array(
4203		- array('text' => $text)
4204		- )
4205		- )
4206		- );
	2484	+ if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
	2485	+ //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
	2486	+ // Consider truncating text here
	2487	+ }
4207	2488
4208		- // Set output dimensionality to 1536 for consistency with other models
4209		- $request_body['outputDimensionality'] = 1536;
4210		- } else {
4211		- // OpenAI/Voyage API format
4212		- $request_body = array(
4213		- 'model' => $selected_model,
4214		- 'input' => $text
4215		- );
	2489	+ // Prepare request body based on provider
	2490	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2491	+ // Gemini API format
	2492	+ $request_body = array(
	2493	+ 'model' => 'models/' . $selected_model,
	2494	+ 'content' => array(
	2495	+ 'parts' => array(
	2496	+ array('text' => $text)
	2497	+ )
	2498	+ )
	2499	+ );
4216	2500
4217		- // Add output_dimension for voyage-3-large model
4218		- if ($selected_model === 'voyage-3-large') {
4219		- $request_body['output_dimension'] = 2048;
4220		- }
4221		- }
	2501	+ // Set output dimensionality to 1536 for consistency with other models
	2502	+ $request_body['outputDimensionality'] = 1536;
	2503	+ } else {
	2504	+ // OpenAI/Voyage API format
	2505	+ $request_body = array(
	2506	+ 'model' => $selected_model,
	2507	+ 'input' => $text
	2508	+ );
4222	2509
4223		- //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
	2510	+ // Add output_dimension for voyage-3-large model
	2511	+ if ($selected_model === 'voyage-3-large') {
	2512	+ $request_body['output_dimension'] = 2048;
	2513	+ }
	2514	+ }
4224	2515
4225		- // Prepare headers based on provider
4226		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4227		- // Gemini uses API key as query parameter
4228		- $endpoint .= '?key=' . $api_key;
4229		- $headers = array(
4230		- 'Content-Type' => 'application/json'
4231		- );
4232		- } else {
4233		- // OpenAI/Voyage use Bearer token
4234		- $headers = array(
4235		- 'Authorization' => 'Bearer ' . $api_key,
4236		- 'Content-Type' => 'application/json'
4237		- );
4238		- }
	2516	+ //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
4239	2517
4240		- // Make API request
4241		- //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
4242		- $response = wp_remote_post($endpoint, array(
4243		- 'body' => wp_json_encode($request_body),
4244		- 'headers' => $headers,
4245		- 'timeout' => 60 // Increased timeout for large inputs
4246		- ));
	2518	+ // Prepare headers based on provider
	2519	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2520	+ // Gemini uses API key as query parameter
	2521	+ $endpoint .= '?key=' . $api_key;
	2522	+ $headers = array(
	2523	+ 'Content-Type' => 'application/json'
	2524	+ );
	2525	+ } else {
	2526	+ // OpenAI/Voyage use Bearer token
	2527	+ $headers = array(
	2528	+ 'Authorization' => 'Bearer ' . $api_key,
	2529	+ 'Content-Type' => 'application/json'
	2530	+ );
	2531	+ }
4247	2532
4248		- // Handle wp_remote_post errors
4249		- if (is_wp_error($response)) {
4250		- $error_message = $response->get_error_message();
4251		- //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
4252		- return 'Connection error: ' . $error_message;
4253		- }
	2533	+ // Make API request
	2534	+ //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
	2535	+ $response = wp_remote_post($endpoint, array(
	2536	+ 'body' => wp_json_encode($request_body),
	2537	+ 'headers' => $headers,
	2538	+ 'timeout' => 60 // Increased timeout for large inputs
	2539	+ ));
4254	2540
4255		- // Get and check HTTP response code
4256		- $http_code = wp_remote_retrieve_response_code($response);
4257		- //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
	2541	+ // Handle wp_remote_post errors
	2542	+ if (is_wp_error($response)) {
	2543	+ $error_message = $response->get_error_message();
	2544	+ //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
	2545	+ return 'Connection error: ' . $error_message;
	2546	+ }
4258	2547
4259		- if ($http_code !== 200) {
4260		- $error_body = wp_remote_retrieve_body($response);
4261		- //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
	2548	+ // Get and check HTTP response code
	2549	+ $http_code = wp_remote_retrieve_response_code($response);
	2550	+ //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
4262	2551
4263		- // Try to parse error for more details
4264		- $error_json = json_decode($error_body, true);
4265		- if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
4266		- $error_type = $error_json['error']['type'] ?? 'unknown';
4267		- $error_message = $error_json['error']['message'] ?? 'No message';
4268		- //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
4269		- //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
	2552	+ if ($http_code !== 200) {
	2553	+ $error_body = wp_remote_retrieve_body($response);
	2554	+ //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
4270	2555
4271		- // Customize error message for common API errors
4272		- if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
4273		- $error_message = sprintf('Invalid %s API key for bot %s. Please check your API key in the bot settings.', $provider_name, $bot_id);
4274		- } elseif ($error_type === 'authentication_error') {
4275		- $error_message = sprintf('%s authentication failed for bot %s. Please verify your API key in the bot settings.', $provider_name, $bot_id);
4276		- }
	2556	+ // Try to parse error for more details
	2557	+ $error_json = json_decode($error_body, true);
	2558	+ if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
	2559	+ $error_type = $error_json['error']['type'] ?? 'unknown';
	2560	+ $error_message = $error_json['error']['message'] ?? 'No message';
	2561	+ //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
	2562	+ //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
4277	2563
4278		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4279		- return $error_message;
4280		- }
	2564	+ // Customize error message for common API errors
	2565	+ if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
	2566	+ $error_message = sprintf('Invalid %s API key. Please check your API key in the MxChat settings.', $provider_name);
	2567	+ } elseif ($error_type === 'authentication_error') {
	2568	+ $error_message = sprintf('%s authentication failed. Please verify your API key in the MxChat settings.', $provider_name);
	2569	+ }
4281	2570
4282		- $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding for bot %s", $http_code, $bot_id);
4283		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4284		- return $error_message;
4285		- }
	2571	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2572	+ return $error_message;
	2573	+ }
4286	2574
4287		- // Parse response body
4288		- $response_body = wp_remote_retrieve_body($response);
4289		- //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
	2575	+ $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding", $http_code);
	2576	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2577	+ return $error_message;
	2578	+ }
4290	2579
4291		- $response_data = json_decode($response_body, true);
	2580	+ // Parse response body
	2581	+ $response_body = wp_remote_retrieve_body($response);
	2582	+ //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
4292	2583
4293		- if (json_last_error() !== JSON_ERROR_NONE) {
4294		- $error = json_last_error_msg();
4295		- //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
4296		- //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
4297		- return "Failed to parse API response: $error";
4298		- }
	2584	+ $response_data = json_decode($response_body, true);
4299	2585
4300		- // Handle different response formats based on provider
4301		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4302		- // Gemini API response format
4303		- if (isset($response_data['embedding']['values'])) {
4304		- $embedding_dimensions = count($response_data['embedding']['values']);
4305		- //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
	2586	+ if (json_last_error() !== JSON_ERROR_NONE) {
	2587	+ $error = json_last_error_msg();
	2588	+ //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
	2589	+ //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
	2590	+ return "Failed to parse API response: $error";
	2591	+ }
4306	2592
4307		- // Check if embedding dimensions are as expected (should be 1536)
4308		- if ($embedding_dimensions !== 1536) {
4309		- //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
4310		- }
	2593	+ // Handle different response formats based on provider
	2594	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2595	+ // Gemini API response format
	2596	+ if (isset($response_data['embedding']['values'])) {
	2597	+ $embedding_dimensions = count($response_data['embedding']['values']);
	2598	+ //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
4311	2599
4312		- return $response_data['embedding']['values'];
4313		- } else {
4314		- //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
4315		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2600	+ // Check if embedding dimensions are as expected (should be 1536)
	2601	+ if ($embedding_dimensions !== 1536) {
	2602	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
	2603	+ }
4316	2604
4317		- if (isset($response_data['error'])) {
4318		- $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
4319		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4320		- return $error_message;
4321		- }
	2605	+ return $response_data['embedding']['values'];
	2606	+ } else {
	2607	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
	2608	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4322	2609
4323		- $error_message = "Invalid Gemini API response format: No embedding found";
4324		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4325		- return $error_message;
4326		- }
4327		- } else {
4328		- // OpenAI/Voyage API response format
4329		- if (isset($response_data['data'][0]['embedding'])) {
4330		- $embedding_dimensions = count($response_data['data'][0]['embedding']);
4331		- //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
	2610	+ if (isset($response_data['error'])) {
	2611	+ $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
	2612	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2613	+ return $error_message;
	2614	+ }
4332	2615
4333		- // Check if embedding dimensions are as expected
4334		- if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
4335		- ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
4336		- //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
4337		- }
	2616	+ $error_message = "Invalid Gemini API response format: No embedding found";
	2617	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2618	+ return $error_message;
	2619	+ }
	2620	+ } else {
	2621	+ // OpenAI/Voyage API response format
	2622	+ if (isset($response_data['data'][0]['embedding'])) {
	2623	+ $embedding_dimensions = count($response_data['data'][0]['embedding']);
	2624	+ //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
4338	2625
4339		- return $response_data['data'][0]['embedding'];
4340		- } else {
4341		- //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
4342		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2626	+ // Check if embedding dimensions are as expected
	2627	+ if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
	2628	+ ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
	2629	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
	2630	+ }
4343	2631
4344		- if (isset($response_data['error'])) {
4345		- $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
4346		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4347		- return $error_message;
4348		- }
	2632	+ return $response_data['data'][0]['embedding'];
	2633	+ } else {
	2634	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
	2635	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4349	2636
4350		- $error_message = "Invalid API response format: No embedding found";
4351		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4352		- return $error_message;
4353		- }
4354		- }
4355		-}
	2637	+ if (isset($response_data['error'])) {
	2638	+ $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
	2639	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2640	+ return $error_message;
	2641	+ }
4356	2642
4357		-/**
4358		- * Get bot-specific options for multi-bot functionality
4359		- * Falls back to default options if bot_id is 'default' or multi-bot add-on is not active
4360		- */
4361		-private function get_bot_options($bot_id = 'default') {
4362		- //error_log("MXCHAT DEBUG: get_bot_options called for bot: " . $bot_id);
4363		-
4364		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4365		- //error_log("MXCHAT DEBUG: Using default options (no multi-bot or bot is 'default')");
4366		- return array();
4367		- }
4368		-
4369		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
4370		-
4371		- if (!empty($bot_options)) {
4372		- //error_log("MXCHAT DEBUG: Got bot-specific options from filter");
4373		- if (isset($bot_options['similarity_threshold'])) {
4374		- //error_log(" - similarity_threshold: " . $bot_options['similarity_threshold']);
4375		- }
4376		- }
4377		-
4378		- return is_array($bot_options) ? $bot_options : array();
4379		-}
4380		-
4381		-/**
4382		- * Get bot-specific Pinecone configuration
4383		- * Used in the knowledge retrieval functions
4384		- */
4385		-// Also add debugging to your get_bot_pinecone_config function
4386		-private function get_bot_pinecone_config($bot_id = 'default') {
4387		- //error_log("MXCHAT DEBUG: get_bot_pinecone_config called for bot: " . $bot_id);
4388		-
4389		- // If default bot or multi-bot add-on not active, use default Pinecone config
4390		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4391		- //error_log("MXCHAT DEBUG: Using default Pinecone config (no multi-bot or bot is 'default')");
4392		- $addon_options = get_option('mxchat_pinecone_addon_options', array());
4393		- $config = array(
4394		- 'use_pinecone' => (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1'),
4395		- 'api_key' => $addon_options['mxchat_pinecone_api_key'] ?? '',
4396		- 'host' => $addon_options['mxchat_pinecone_host'] ?? '',
4397		- 'namespace' => $addon_options['mxchat_pinecone_namespace'] ?? ''
4398		- );
4399		- //error_log("MXCHAT DEBUG: Default config - use_pinecone: " . ($config['use_pinecone'] ? 'true' : 'false'));
4400		- return $config;
4401		- }
4402		-
4403		- //error_log("MXCHAT DEBUG: Calling filter 'mxchat_get_bot_pinecone_config' for bot: " . $bot_id);
4404		-
4405		- // Hook for multi-bot add-on to provide bot-specific Pinecone config
4406		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
4407		-
4408		- if (!empty($bot_pinecone_config)) {
4409		- //error_log("MXCHAT DEBUG: Got bot-specific config from filter");
4410		- //error_log(" - use_pinecone: " . (isset($bot_pinecone_config['use_pinecone']) ? ($bot_pinecone_config['use_pinecone'] ? 'true' : 'false') : 'not set'));
4411		- //error_log(" - host: " . ($bot_pinecone_config['host'] ?? 'not set'));
4412		- //error_log(" - namespace: " . ($bot_pinecone_config['namespace'] ?? 'not set'));
4413		- } else {
4414		- //error_log("MXCHAT DEBUG: Filter returned empty config!");
4415		- }
4416		-
4417		- return is_array($bot_pinecone_config) ? $bot_pinecone_config : array();
4418		-}
4419		-
4420		-
	2643	+ $error_message = "Invalid API response format: No embedding found";
	2644	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2645	+ return $error_message;
	2646	+ }
	2647	+ }
	2648	+ }
4421	2649	public function mxchat_ajax_dismiss_completed_status() {
4422	2650	try {
4423	2651	// Verify the request
4424	2652	check_ajax_referer('mxchat_status_nonce', 'nonce');
		@@ -4781,375 +3009,18 @@
4781	3009
4782	3010	return $html;
4783	3011	}
4784	3012
4785		-/**
4786		- * Get all ACF fields for a specific post, excluding any fields the user has disabled
4787		- */
4788		-public function mxchat_get_acf_fields_for_post($post_id) {
4789		- if (!function_exists('get_fields')) {
4790		- return array();
4791		- }
4792		-
4793		- $fields = get_fields($post_id);
4794		- if (!$fields || !is_array($fields)) {
4795		- return array();
4796		- }
4797		-
4798		- // Get excluded fields from settings
4799		- $excluded_fields = get_option('mxchat_acf_excluded_fields', array());
4800		- if (!empty($excluded_fields) && is_array($excluded_fields)) {
4801		- foreach ($excluded_fields as $excluded_field) {
4802		- if (isset($fields[$excluded_field])) {
4803		- unset($fields[$excluded_field]);
4804		- }
4805		- }
4806		- }
4807		-
4808		- return $fields;
4809		-}
4810		-
4811		-/**
4812		- * Get all registered ACF field groups and their fields for the settings UI
4813		- */
4814		-public function mxchat_get_all_acf_fields() {
4815		- if (!function_exists('acf_get_field_groups') || !function_exists('acf_get_fields')) {
4816		- return array();
4817		- }
4818		-
4819		- $all_fields = array();
4820		- $field_groups = acf_get_field_groups();
4821		-
4822		- if (!empty($field_groups)) {
4823		- foreach ($field_groups as $group) {
4824		- $group_fields = acf_get_fields($group['key']);
4825		- if (!empty($group_fields)) {
4826		- $all_fields[$group['title']] = array();
4827		- foreach ($group_fields as $field) {
4828		- $all_fields[$group['title']][] = array(
4829		- 'name' => $field['name'],
4830		- 'label' => $field['label'],
4831		- 'type' => $field['type']
4832		- );
4833		- }
4834		- }
4835		- }
4836		- }
4837		-
4838		- return $all_fields;
4839		-}
4840		-
4841		-/**
4842		- * Get whitelisted custom post meta for a given post
4843		- * This allows non-ACF meta fields (like OptionTree, theme meta boxes) to be included in embeddings
4844		- */
4845		-public function mxchat_get_whitelisted_post_meta($post_id) {
4846		- $whitelist = get_option('mxchat_custom_meta_whitelist', '');
4847		-
4848		- if (empty($whitelist)) {
4849		- return array();
4850		- }
4851		-
4852		- // Parse the whitelist - one meta key per line
4853		- $meta_keys = array_filter(array_map('trim', explode("\n", $whitelist)));
4854		-
4855		- if (empty($meta_keys)) {
4856		- return array();
4857		- }
4858		-
4859		- $result = array();
4860		-
4861		- foreach ($meta_keys as $key) {
4862		- // Skip empty keys
4863		- if (empty($key)) {
4864		- continue;
4865		- }
4866		-
4867		- $value = get_post_meta($post_id, $key, true);
4868		-
4869		- // Only include non-empty string values
4870		- if (!empty($value) && is_string($value)) {
4871		- $result[$key] = $value;
4872		- } elseif (!empty($value) && is_array($value)) {
4873		- // Handle array values by joining them
4874		- $flat_value = $this->mxchat_flatten_meta_array($value);
4875		- if (!empty($flat_value)) {
4876		- $result[$key] = $flat_value;
4877		- }
4878		- }
4879		- }
4880		-
4881		- return $result;
4882		-}
4883		-
4884		-/**
4885		- * Flatten array meta values into a readable string
4886		- */
4887		-private function mxchat_flatten_meta_array($array, $depth = 0) {
4888		- if ($depth > 3) {
4889		- return ''; // Prevent infinite recursion
4890		- }
4891		-
4892		- $parts = array();
4893		-
4894		- foreach ($array as $key => $value) {
4895		- if (is_string($value) && !empty($value)) {
4896		- $parts[] = $value;
4897		- } elseif (is_array($value)) {
4898		- $nested = $this->mxchat_flatten_meta_array($value, $depth + 1);
4899		- if (!empty($nested)) {
4900		- $parts[] = $nested;
4901		- }
4902		- }
4903		- }
4904		-
4905		- return implode(', ', $parts);
4906		-}
4907		-
4908		-/**
4909		- * Format ACF field values for content extraction
4910		- */
4911		-public function mxchat_format_acf_field_value($value, $field_name = '', $post_id = 0) {
4912		- if (empty($value)) {
4913		- return '';
4914		- }
4915		-
4916		- // Handle WP_Post objects first (THIS IS THE KEY FIX)
4917		- if ($value instanceof WP_Post) {
4918		- return $value->post_title ?: '';
4919		- }
4920		-
4921		- // Handle other WP objects
4922		- if (is_object($value)) {
4923		- if (isset($value->post_title)) {
4924		- return $value->post_title;
4925		- } elseif (isset($value->display_name)) {
4926		- return $value->display_name;
4927		- } elseif (isset($value->name)) {
4928		- return $value->name;
4929		- } elseif (method_exists($value, '__toString')) {
4930		- try {
4931		- return (string) $value;
4932		- } catch (Exception $e) {
4933		- return '';
4934		- }
4935		- }
4936		- // For any other objects, return empty string
4937		- return '';
4938		- }
4939		-
4940		- // Handle different ACF field types
4941		- if (is_array($value)) {
4942		- // Check if it's an image/file field
4943		- if (isset($value['url'])) {
4944		- // Image field - return alt text, title, or caption
4945		- if (!empty($value['alt'])) {
4946		- return $value['alt'];
4947		- } elseif (!empty($value['title'])) {
4948		- return $value['title'];
4949		- } elseif (!empty($value['caption'])) {
4950		- return $value['caption'];
4951		- } else {
4952		- return ''; // Don't include just the URL
4953		- }
4954		- }
4955		-
4956		- // Check if it's a post object or relationship field
4957		- if (isset($value['post_title'])) {
4958		- return $value['post_title'];
4959		- }
4960		-
4961		- // Check if it's a user field
4962		- if (isset($value['display_name'])) {
4963		- return $value['display_name'];
4964		- }
4965		-
4966		- // Check if it's a taxonomy term
4967		- if (isset($value['name']) && isset($value['taxonomy'])) {
4968		- return $value['name'];
4969		- }
4970		-
4971		- // Check if it's a select field with label
4972		- if (isset($value['label'])) {
4973		- return $value['label'];
4974		- }
4975		-
4976		- // Check for repeater field or flexible content
4977		- if (is_numeric(key($value))) {
4978		- $sub_values = array();
4979		- foreach ($value as $sub_item) {
4980		- if (is_array($sub_item)) {
4981		- // For repeater/flexible content, extract text values
4982		- $sub_text = $this->mxchat_extract_text_from_acf_array($sub_item);
4983		- if (!empty($sub_text)) {
4984		- $sub_values[] = $sub_text;
4985		- }
4986		- } elseif ($sub_item instanceof WP_Post) {
4987		- // Handle WP_Post objects in arrays
4988		- $sub_values[] = $sub_item->post_title ?: '';
4989		- } else {
4990		- $sub_values[] = (string) $sub_item;
4991		- }
4992		- }
4993		- return implode(', ', array_filter($sub_values));
4994		- }
4995		-
4996		- // For other arrays, try to extract meaningful text
4997		- $text_values = array();
4998		- foreach ($value as $key => $val) {
4999		- if (is_string($val) && !empty(trim($val))) {
5000		- $text_values[] = trim($val);
5001		- } elseif ($val instanceof WP_Post) {
5002		- // Handle WP_Post objects in associative arrays
5003		- $text_values[] = $val->post_title ?: '';
5004		- } elseif (is_array($val) && isset($val['post_title'])) {
5005		- $text_values[] = $val['post_title'];
5006		- } elseif (is_array($val) && isset($val['name'])) {
5007		- $text_values[] = $val['name'];
5008		- }
5009		- }
5010		-
5011		- return implode(', ', array_filter($text_values));
5012		- }
5013		-
5014		- // Handle boolean values
5015		- if (is_bool($value)) {
5016		- return $value ? 'Yes' : 'No';
5017		- }
5018		-
5019		- // Handle numeric values
5020		- if (is_numeric($value)) {
5021		- return (string) $value;
5022		- }
5023		-
5024		- // Handle string values
5025		- if (is_string($value)) {
5026		- return trim($value);
5027		- }
5028		-
5029		- // For anything else that we can't handle, return empty string
5030		- // This prevents the "Object could not be converted to string" error
5031		- return '';
5032		-}
5033		-
5034		-/**
5035		- * Extract text from complex ACF array structures
5036		- */
5037		-private function mxchat_extract_text_from_acf_array($array) {
5038		- if (!is_array($array)) {
5039		- return '';
5040		- }
5041		-
5042		- $text_parts = array();
5043		-
5044		- foreach ($array as $key => $value) {
5045		- if (is_string($value) && !empty(trim($value))) {
5046		- // Skip keys that are likely to be IDs or technical values
5047		- if (!is_numeric($value) || strlen($value) > 10) {
5048		- $text_parts[] = trim($value);
5049		- }
5050		- } elseif ($value instanceof WP_Post) {
5051		- // Handle WP_Post objects
5052		- $text_parts[] = $value->post_title ?: '';
5053		- } elseif (is_array($value)) {
5054		- if (isset($value['post_title'])) {
5055		- $text_parts[] = $value['post_title'];
5056		- } elseif (isset($value['name'])) {
5057		- $text_parts[] = $value['name'];
5058		- } elseif (isset($value['label'])) {
5059		- $text_parts[] = $value['label'];
5060		- }
5061		- } elseif (is_object($value)) {
5062		- // Handle other objects safely
5063		- if (isset($value->post_title)) {
5064		- $text_parts[] = $value->post_title;
5065		- } elseif (isset($value->name)) {
5066		- $text_parts[] = $value->name;
5067		- } elseif (isset($value->display_name)) {
5068		- $text_parts[] = $value->display_name;
5069		- }
5070		- }
5071		- }
5072		-
5073		- return implode(', ', array_filter($text_parts));
5074		-}
5075		-
5076		-/**
5077		- * Handle ACF save - fires after ACF fields are saved
5078		- * This ensures ACF field data is available when syncing to knowledge base
5079		- */
5080		-public function mxchat_handle_acf_save($post_id) {
5081		- // Skip if not a valid post
5082		- if (!$post_id || $post_id === 'options') {
5083		- return;
5084		- }
5085		-
5086		- // Skip autosaves and revisions
	3013	+public function mxchat_handle_post_update($post_id, $post, $update) {
	3014	+ // Basic validation checks
5087	3015	if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5088	3016	return;
5089	3017	}
5090		-
5091		- $post = get_post($post_id);
5092		- if (!$post) {
5093		- return;
5094		- }
5095		-
5096		- $post_type = $post->post_type;
5097		-
5098		- // Check if sync is enabled for this post type
5099		- $should_sync = false;
5100		-
5101		- if ($post_type === 'post' && get_option('mxchat_auto_sync_posts') === '1') {
5102		- $should_sync = true;
5103		- } else if ($post_type === 'page' && get_option('mxchat_auto_sync_pages') === '1') {
5104		- $should_sync = true;
5105		- } else if ($post_type === 'product' && class_exists('WooCommerce')) {
5106		- // WooCommerce products - check if WooCommerce integration is enabled
5107		- $options = get_option('mxchat_options', array());
5108		- if (isset($options['enable_woocommerce_integration']) &&
5109		- ($options['enable_woocommerce_integration'] === '1' || $options['enable_woocommerce_integration'] === 'on')) {
5110		- $should_sync = true;
5111		- }
5112		- } else {
5113		- // Check custom post types
5114		- $option_name = 'mxchat_auto_sync_' . $post_type;
5115		- if (get_option($option_name) === '1') {
5116		- $should_sync = true;
5117		- }
5118		- }
5119		-
5120		- if (!$should_sync) {
5121		- return;
5122		- }
5123		-
5124		- // Only process published posts
	3018	+
	3019	+ // Only process published content
5125	3020	if ($post->post_status !== 'publish') {
5126	3021	return;
5127	3022	}
5128		-
5129		- // Check if this post has any ACF fields - if not, no need to re-sync
5130		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
5131		- if (empty($acf_fields)) {
5132		- return;
5133		- }
5134		-
5135		- // Use a transient to prevent duplicate processing (post_updated may have already run)
5136		- $transient_key = 'mxchat_acf_synced_' . $post_id;
5137		- if (get_transient($transient_key)) {
5138		- return;
5139		- }
5140		- set_transient($transient_key, true, 60); // Prevent re-processing for 60 seconds
5141		-
5142		- // Re-run the sync with ACF data now available
5143		- // We pass $update=true since this is effectively an update with ACF data
5144		- $this->mxchat_handle_post_update($post_id, $post, true);
5145		-}
5146		-
5147		-public function mxchat_handle_post_update($post_id, $post, $update) {
5148		- // Basic validation checks
5149		- if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5150		- return;
5151		- }
5152	3023
5153	3024	$post_type = $post->post_type;
5154	3025
5155	3026	// Check if sync is enabled for this post type
		@@ -5171,244 +3042,80 @@
5171	3042	if (!$should_sync) {
5172	3043	return;
5173	3044	}
5174	3045
5175		- // Check if we have stored the previous status and URL in our transients
5176		- $previous_status_key = 'mxchat_prev_status_' . $post_id;
5177		- $previous_status = get_transient($previous_status_key);
	3046	+ // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
	3047	+ $title = get_the_title($post_id);
	3048	+ $content = get_post_field('post_content', $post_id);
5178	3049
5179		- $previous_url_key = 'mxchat_prev_url_' . $post_id;
5180		- $previous_url = get_transient($previous_url_key);
	3050	+ // Apply WordPress content filters to get properly formatted content
	3051	+ $content = apply_filters('the_content', $content);
5181	3052
5182		- // If the post was previously published but is now not published, remove from knowledge base
5183		- if ($previous_status === 'publish' && $post->post_status !== 'publish') {
5184		- // Use the stored URL from when it was published, or fall back to current permalink
5185		- $source_url = $previous_url ?: get_permalink($post_id);
5186		-
5187		- if ($source_url) {
5188		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
5189		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	3053	+ // Strip tags but preserve structure
	3054	+ $content = wp_strip_all_tags($content);
	3055	+
	3056	+ // Combine title and content
	3057	+ $final_content = $title . "\n\n" . $content;
	3058	+
	3059	+ // For custom post types like job_listing, include additional fields
	3060	+ if ($post_type === 'job_listing') {
	3061	+ // Add job-specific meta if available
	3062	+ $job_location = get_post_meta($post_id, '_job_location', true);
	3063	+ if (!empty($job_location)) {
	3064	+ $final_content .= "\n\nLocation: " . $job_location;
5190	3065	}
5191		-
5192		- // Clean up the transients and exit early
5193		- delete_transient($previous_status_key);
5194		- delete_transient($previous_url_key);
5195		- return;
5196		- }
5197		-
5198		- // Slug/permalink rename while still published: delete the old vectors before upserting new ones.
5199		- // Without this, md5(old_url) vectors (base + chunks) would be orphaned under the stale URL.
5200		- if ($post->post_status === 'publish' && !empty($previous_url)) {
5201		- $current_url = get_permalink($post_id);
5202		- if ($current_url && $current_url !== $previous_url) {
5203		- MxChat_Utils::delete_chunks_for_url($previous_url, 'default');
5204		- }
5205		- }
5206		-
5207		- // Store the current status for next time (if this is an update)
5208		- if ($update) {
5209		- set_transient($previous_status_key, $post->post_status, DAY_IN_SECONDS);
5210		-
5211		- // If the post is currently published, also store its URL
5212		- if ($post->post_status === 'publish') {
5213		- $current_url = get_permalink($post_id);
5214		- set_transient($previous_url_key, $current_url, DAY_IN_SECONDS);
5215		- }
5216		- }
5217		-
5218		- // Only process currently published content for adding/updating
5219		- if ($post->post_status === 'publish') {
5220		- // Get the source URL
5221		- $source_url = get_permalink($post_id);
5222	3066
5223		- // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
5224		- $title = get_the_title($post_id);
5225		- $content = get_post_field('post_content', $post_id);
5226		- $excerpt = get_post_field('post_excerpt', $post_id);
5227		-
5228		- // Remove shortcode tags but preserve content inside them
5229		- $content = $this->strip_shortcode_tags_preserve_content($content);
5230		- $excerpt = $this->strip_shortcode_tags_preserve_content($excerpt);
5231		-
5232		- // Strip tags but preserve structure (don't use 'the_content' filter as it may re-add shortcodes)
5233		- $content = wp_strip_all_tags($content);
5234		-
5235		- // Combine title, short description (if exists), and content
5236		- $final_content = $title . "\n\n";
5237		-
5238		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
5239		- if (!empty($excerpt)) {
5240		- $final_content .= "Short Description: " . wp_strip_all_tags($excerpt) . "\n\n";
5241		- }
5242		-
5243		- $final_content .= $content;
5244		-
5245		- // For WooCommerce products, include pricing and product details
5246		- if ($post_type === 'product' && class_exists('WooCommerce')) {
5247		- $product = wc_get_product($post_id);
5248		-
5249		- if ($product) {
5250		- // Get pricing information
5251		- $regular_price = $product->get_regular_price();
5252		- $sale_price = $product->get_sale_price();
5253		- $price = $product->get_price();
5254		- $sku = $product->get_sku();
5255		-
5256		- // Get currency symbol
5257		- $currency_symbol = get_woocommerce_currency_symbol();
5258		-
5259		- // Add pricing information
5260		- $final_content .= "\n";
5261		- if (!empty($regular_price)) {
5262		- $final_content .= "Price: " . $currency_symbol . $regular_price . "\n";
5263		- } elseif (!empty($price)) {
5264		- $final_content .= "Price: " . $currency_symbol . $price . "\n";
5265		- }
5266		-
5267		- if (!empty($sale_price) && $sale_price !== $regular_price) {
5268		- $final_content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
5269		- }
5270		-
5271		- // Handle variable products - show price range
5272		- if ($product->is_type('variable')) {
5273		- $min_price = $product->get_variation_price('min');
5274		- $max_price = $product->get_variation_price('max');
5275		- if ($min_price !== $max_price) {
5276		- $final_content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
5277		- }
5278		- }
5279		-
5280		- if (!empty($sku)) {
5281		- $final_content .= "SKU: " . $sku . "\n";
5282		- }
5283		-
5284		- // Get product categories
5285		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
5286		- if (!empty($categories) && !is_wp_error($categories)) {
5287		- $final_content .= "Categories: " . implode(', ', $categories) . "\n";
5288		- }
	3067	+ // Get job type terms
	3068	+ $job_types = get_the_terms($post_id, 'job_listing_type');
	3069	+ if (!empty($job_types) && !is_wp_error($job_types)) {
	3070	+ $types = array();
	3071	+ foreach ($job_types as $type) {
	3072	+ $types[] = $type->name;
5289	3073	}
	3074	+ $final_content .= "\n\nJob Type: " . implode(', ', $types);
5290	3075	}
5291		-
5292		- // For custom post types like job_listing, include additional fields
5293		- if ($post_type === 'job_listing') {
5294		- // Add job-specific meta if available
5295		- $job_location = get_post_meta($post_id, '_job_location', true);
5296		- if (!empty($job_location)) {
5297		- $final_content .= "\n\nLocation: " . $job_location;
5298		- }
5299		-
5300		- // Get job type terms
5301		- $job_types = get_the_terms($post_id, 'job_listing_type');
5302		- if (!empty($job_types) && !is_wp_error($job_types)) {
5303		- $types = array();
5304		- foreach ($job_types as $type) {
5305		- $types[] = $type->name;
5306		- }
5307		- $final_content .= "\n\nJob Type: " . implode(', ', $types);
5308		- }
5309		-
5310		- // Get company name if available
5311		- $company_name = get_post_meta($post_id, '_company_name', true);
5312		- if (!empty($company_name)) {
5313		- $final_content .= "\n\nCompany: " . $company_name;
5314		- }
5315		- }
5316		-
5317		- // ADD ACF FIELDS SUPPORT (matches ajax_mxchat_process_selected_content behavior)
5318		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
5319		- if (!empty($acf_fields)) {
5320		- $acf_content_parts = array();
5321		-
5322		- foreach ($acf_fields as $field_name => $field_value) {
5323		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
5324		- if (!empty($formatted_value)) {
5325		- // Convert field name to readable label
5326		- $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name));
5327		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
5328		- }
5329		- }
5330		-
5331		- if (!empty($acf_content_parts)) {
5332		- $final_content .= "\n\n" . implode("\n", $acf_content_parts);
5333		- }
5334		- }
5335		-
5336		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
5337		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
5338		- if (!empty($custom_meta)) {
5339		- $meta_content_parts = array();
5340		-
5341		- foreach ($custom_meta as $meta_key => $meta_value) {
5342		- // Convert meta key to readable label
5343		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
5344		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
5345		- }
5346		-
5347		- if (!empty($meta_content_parts)) {
5348		- $final_content .= "\n\n" . implode("\n", $meta_content_parts);
5349		- }
5350		- }
5351		-
5352		- // Get API key with proper model detection
5353		- $options = get_option('mxchat_options');
5354		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
5355	3076
5356		- if (strpos($selected_model, 'voyage') === 0) {
5357		- $api_key = $options['voyage_api_key'] ?? '';
5358		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
5359		- $api_key = $options['gemini_api_key'] ?? '';
5360		- } else {
5361		- $api_key = $options['api_key'] ?? '';
	3077	+ // Get company name if available
	3078	+ $company_name = get_post_meta($post_id, '_company_name', true);
	3079	+ if (!empty($company_name)) {
	3080	+ $final_content .= "\n\nCompany: " . $company_name;
5362	3081	}
5363		-
5364		- if (empty($api_key)) {
5365		- return;
5366		- }
5367		-
5368		- // Use the centralized utility function for storage
5369		- $result = MxChat_Utils::submit_content_to_db(
5370		- $final_content,
5371		- $source_url,
5372		- $api_key,
5373		- md5($source_url) // Vector ID for Pinecone
5374		- );
5375		-
5376		- // After successful storage, apply role restriction based on tags
5377		- if (!is_wp_error($result)) {
5378		- $this->apply_role_restriction_to_post($post_id, $source_url);
5379		- }
5380	3082	}
5381	3083
5382		- // Clean up the stored previous status if not used above
5383		- if ($previous_status !== 'publish' || $post->post_status === 'publish') {
5384		- delete_transient($previous_status_key);
5385		- delete_transient($previous_url_key);
	3084	+ // Get the source URL
	3085	+ $source_url = get_permalink($post_id);
	3086	+
	3087	+ // Get API key with proper model detection
	3088	+ $options = get_option('mxchat_options');
	3089	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	3090	+
	3091	+ if (strpos($selected_model, 'voyage') === 0) {
	3092	+ $api_key = $options['voyage_api_key'] ?? '';
	3093	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	3094	+ $api_key = $options['gemini_api_key'] ?? '';
	3095	+ } else {
	3096	+ $api_key = $options['api_key'] ?? '';
5386	3097	}
5387		-}
5388		-
5389		-/**
5390		- * Store the post status and URL before update to detect status transitions
5391		- * This runs before the post is actually updated in the database
5392		- */
5393		-public function mxchat_store_pre_update_status($post_id, $data) {
5394		- // Get the current post from database (before update)
5395		- $current_post = get_post($post_id);
5396	3098
5397		- if ($current_post) {
5398		- // Store the current status temporarily
5399		- $status_key = 'mxchat_prev_status_' . $post_id;
5400		- set_transient($status_key, $current_post->post_status, HOUR_IN_SECONDS);
5401		-
5402		- // If the post is currently published, also store its URL
5403		- if ($current_post->post_status === 'publish') {
5404		- $url_key = 'mxchat_prev_url_' . $post_id;
5405		- $current_url = get_permalink($post_id);
5406		- set_transient($url_key, $current_url, HOUR_IN_SECONDS);
5407		- }
	3099	+ if (empty($api_key)) {
	3100	+ //error_log('MxChat Auto-sync: No API key configured for embedding model');
	3101	+ return;
5408	3102	}
	3103	+
	3104	+ // Use the centralized utility function for storage
	3105	+ $result = MxChat_Utils::submit_content_to_db(
	3106	+ $final_content,
	3107	+ $source_url,
	3108	+ $api_key,
	3109	+ md5($source_url) // Vector ID for Pinecone
	3110	+ );
	3111	+
	3112	+ if (is_wp_error($result)) {
	3113	+ //error_log('MxChat Auto-sync failed for post ' . $post_id . ': ' . $result->get_error_message());
	3114	+ }
5409	3115	}
5410	3116
	3117	+
5411	3118	public function mxchat_handle_post_delete($post_id) {
5412	3119	// Get post data before it's deleted
5413	3120	$post = get_post($post_id);
5414	3121
		@@ -5438,51 +3145,85 @@
5438	3145	if (!$should_sync) {
5439	3146	return;
5440	3147	}
5441	3148
5442		- // Resolve the pre-trash URL. wp_trash_post renames the slug with "__trashed" before firing
5443		- // this hook, so get_permalink() here would return the trashed URL and md5() would miss the
5444		- // real vector IDs stored under the original URL.
5445		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
	3149	+ // Get the URL before post is deleted
	3150	+ $source_url = get_permalink($post_id);
5446	3151	if (!$source_url) {
5447		- //error_log('MXChat: Failed to resolve source URL for post ' . $post_id);
	3152	+ //error_log('MXChat: Failed to get permalink for post ' . $post_id);
5448	3153	return;
5449	3154	}
5450	3155
5451		- // Use chunk-aware deletion (handles both chunked and non-chunked content)
5452		- $delete_result = MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	3156	+ // Check if Pinecone is enabled
	3157	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	3158	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5453	3159
5454		- if (is_wp_error($delete_result)) {
5455		- //error_log('MXChat: Chunk-aware deletion failed for URL: ' . $source_url . ' - ' . $delete_result->get_error_message());
	3160	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3161	+ // Delete from Pinecone
	3162	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
	3163	+ } else {
	3164	+ // Delete from WordPress DB
	3165	+ global $wpdb;
	3166	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	3167	+
	3168	+ $result = $wpdb->delete(
	3169	+ $table_name,
	3170	+ array('source_url' => $source_url),
	3171	+ array('%s')
	3172	+ );
	3173	+
	3174	+ if ($result === false) {
	3175	+ //error_log('MXChat: WordPress DB deletion failed for URL: ' . $source_url);
	3176	+ }
5456	3177	}
	3178	+}
5457	3179
5458		- delete_transient('mxchat_prev_url_' . $post_id);
5459		- delete_transient('mxchat_prev_status_' . $post_id);
5460		-}
5461	3180
5462		-/**
5463		- * Resolve the source URL for a post being trashed/deleted.
5464		- *
5465		- * Why: wp_trash_post appends "__trashed" to the slug before the wp_trash_post action fires, so
5466		- * get_permalink() returns a URL whose md5() won't match the vector IDs stored in Pinecone or
5467		- * the source_url rows in the WP DB. Prefer the URL captured by mxchat_store_pre_update_status
5468		- * (runs on pre_post_update, before the rename); fall back to stripping the __trashed suffix.
5469		- */
5470		-private function mxchat_resolve_pre_trash_url($post_id) {
5471		- $previous_url = get_transient('mxchat_prev_url_' . $post_id);
5472		- if (!empty($previous_url)) {
5473		- return $previous_url;
5474		- }
	3181	+ /**
	3182	+ * Deletes data from Pinecone using a source URL
	3183	+ */
	3184	+ public function mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options) {
	3185	+ $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
	3186	+ $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
5475	3187
5476		- $current = get_permalink($post_id);
5477		- if (!$current) {
5478		- return '';
5479		- }
5480		- return preg_replace('#__trashed(/?)$#', '$1', $current);
5481		-}
	3188	+ if (empty($host) || empty($api_key)) {
	3189	+ //error_log('MXChat: Pinecone deletion failed - missing configuration');
	3190	+ return false;
	3191	+ }
5482	3192
	3193	+ $api_endpoint = "https://{$host}/vectors/delete";
	3194	+ $vector_id = md5($source_url);
5483	3195
	3196	+ $request_body = array(
	3197	+ 'ids' => array($vector_id)
	3198	+ );
5484	3199
	3200	+ $response = wp_remote_post($api_endpoint, array(
	3201	+ 'headers' => array(
	3202	+ 'Api-Key' => $api_key,
	3203	+ 'accept' => 'application/json',
	3204	+ 'content-type' => 'application/json'
	3205	+ ),
	3206	+ 'body' => wp_json_encode($request_body),
	3207	+ 'timeout' => 30
	3208	+ ));
	3209	+
	3210	+ if (is_wp_error($response)) {
	3211	+ //error_log('MXChat: Pinecone deletion error - ' . $response->get_error_message());
	3212	+ return false;
	3213	+ }
	3214	+
	3215	+ $response_code = wp_remote_retrieve_response_code($response);
	3216	+ if ($response_code !== 200) {
	3217	+ //error_log('MXChat: Pinecone deletion failed with status ' . $response_code);
	3218	+ return false;
	3219	+ }
	3220	+
	3221	+ return true;
	3222	+ }
	3223	+
	3224	+
	3225	+
5485	3226	public function mxchat_handle_product_change($post_id, $post, $update) {
5486	3227	if ($post->post_type !== 'product') {
5487	3228	return;
5488	3229	}
		@@ -5504,12 +3245,11 @@
5504	3245	if (!isset($this->options['enable_woocommerce_integration']) ||
5505	3246	!in_array($this->options['enable_woocommerce_integration'], ['1', 'on'])) {
5506	3247	return;
5507	3248	}
5508		-
	3249	+
5509	3250	$source_url = get_permalink($product->get_id());
5510		- $product_id = $product->get_id();
5511		-
	3251	+
5512	3252	// Build product content
5513	3253	$title = $product->get_name();
5514	3254	$description = $product->get_description();
5515	3255	$short_description = $product->get_short_description();
		@@ -5514,92 +3254,35 @@
5514	3254	$description = $product->get_description();
5515	3255	$short_description = $product->get_short_description();
5516	3256	$regular_price = $product->get_regular_price();
5517	3257	$sale_price = $product->get_sale_price();
5518		- $price = $product->get_price();
5519	3258	$sku = $product->get_sku();
5520		-
5521		- // Get currency symbol
5522		- $currency_symbol = get_woocommerce_currency_symbol();
5523		-
	3259	+
5524	3260	// Format content consistently
5525	3261	$content = $title . "\n\n";
5526		-
5527		- if (!empty($short_description)) {
5528		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
5529		- }
5530		-
	3262	+
5531	3263	if (!empty($description)) {
5532	3264	$content .= wp_strip_all_tags($description) . "\n\n";
5533	3265	}
5534		-
5535		- // Add pricing information
5536		- if (!empty($regular_price)) {
5537		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
5538		- } elseif (!empty($price)) {
5539		- $content .= "Price: " . $currency_symbol . $price . "\n";
	3266	+
	3267	+ if (!empty($short_description)) {
	3268	+ $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
5540	3269	}
5541		-
5542		- if (!empty($sale_price) && $sale_price !== $regular_price) {
5543		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
	3270	+
	3271	+ $content .= "Price: $" . $regular_price . "\n";
	3272	+
	3273	+ if (!empty($sale_price)) {
	3274	+ $content .= "Sale Price: $" . $sale_price . "\n";
5544	3275	}
5545		-
5546		- // Handle variable products - show price range
5547		- if ($product->is_type('variable')) {
5548		- $min_price = $product->get_variation_price('min');
5549		- $max_price = $product->get_variation_price('max');
5550		- if ($min_price !== $max_price) {
5551		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
5552		- }
5553		- }
5554		-
	3276	+
5555	3277	if (!empty($sku)) {
5556	3278	$content .= "SKU: " . $sku . "\n";
5557	3279	}
5558		-
5559		- // Get product categories
5560		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
5561		- if (!empty($categories) && !is_wp_error($categories)) {
5562		- $content .= "Categories: " . implode(', ', $categories) . "\n";
5563		- }
5564		-
5565		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
5566		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
5567		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
5568		- foreach ($custom_tabs as $tab) {
5569		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
5570		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
5571		-
5572		- if (!empty($tab_title) && !empty($tab_content)) {
5573		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
5574		- }
5575		- }
5576		- }
5577		-
5578		- // Also check for reusable/saved tabs applied to this product
5579		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
5580		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
5581		- // Get the saved tabs option
5582		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
5583		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
5584		- foreach ($applied_saved_tabs as $saved_tab_id) {
5585		- if (isset($saved_tabs[$saved_tab_id])) {
5586		- $tab = $saved_tabs[$saved_tab_id];
5587		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
5588		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
5589		-
5590		- if (!empty($tab_title) && !empty($tab_content)) {
5591		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
5592		- }
5593		- }
5594		- }
5595		- }
5596		- }
5597		-
	3280	+
5598	3281	// Get API key with proper model detection
5599	3282	$options = get_option('mxchat_options');
5600	3283	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
5601		-
	3284	+
5602	3285	if (strpos($selected_model, 'voyage') === 0) {
5603	3286	$api_key = $options['voyage_api_key'] ?? '';
5604	3287	} elseif (strpos($selected_model, 'gemini-embedding') === 0) {
5605	3288	$api_key = $options['gemini_api_key'] ?? '';
		@@ -5605,14 +3288,14 @@
5605	3288	$api_key = $options['gemini_api_key'] ?? '';
5606	3289	} else {
5607	3290	$api_key = $options['api_key'] ?? '';
5608	3291	}
5609		-
	3292	+
5610	3293	if (empty($api_key)) {
5611	3294	//error_log('MxChat Auto-sync: No API key configured for embedding model');
5612	3295	return;
5613	3296	}
5614		-
	3297	+
5615	3298	// Use the centralized utility function for storage
5616	3299	$result = MxChat_Utils::submit_content_to_db(
5617	3300	$content,
5618	3301	$source_url,
		@@ -5618,16 +3301,11 @@
5618	3301	$source_url,
5619	3302	$api_key,
5620	3303	md5($source_url) // Vector ID for Pinecone
5621	3304	);
5622		-
5623		- // After successful storage, apply role restriction based on tags
5624		- if (!is_wp_error($result)) {
5625		- $this->apply_role_restriction_to_post($product_id, $source_url);
5626		- }
5627		-
	3305	+
5628	3306	if (is_wp_error($result)) {
5629		- //error_log('MxChat WooCommerce sync failed for product ' . $product_id . ': ' . $result->get_error_message());
	3307	+ //error_log('MxChat WooCommerce sync failed for product ' . $product->get_id() . ': ' . $result->get_error_message());
5630	3308	}
5631	3309	}
5632	3310
5633	3311	public function mxchat_handle_product_delete($post_id) {
		@@ -5634,1112 +3312,30 @@
5634	3312	if (get_post_type($post_id) !== 'product') {
5635	3313	return;
5636	3314	}
5637	3315
5638		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
5639		- if (!$source_url) {
5640		- return;
5641		- }
	3316	+ $source_url = get_permalink($post_id);
5642	3317
5643		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
5644		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
5645		-
5646		- delete_transient('mxchat_prev_url_' . $post_id);
5647		- delete_transient('mxchat_prev_status_' . $post_id);
5648		-}
5649		-
5650		-/**
5651		- * Handle individual Pinecone content deletion
5652		- */
5653		-public function mxchat_handle_pinecone_prompt_delete() {
5654		- // Check permissions
5655		- if (!current_user_can('manage_options')) {
5656		- wp_die(esc_html__('You do not have sufficient permissions.', 'mxchat'));
5657		- }
5658		-
5659		- // Verify nonce
5660		- if (!isset($_GET['_wpnonce']) || !wp_verify_nonce($_GET['_wpnonce'], 'mxchat_delete_pinecone_prompt_nonce')) {
5661		- wp_die(esc_html__('Security check failed.', 'mxchat'));
5662		- }
5663		-
5664		- $vector_id = isset($_GET['vector_id']) ? sanitize_text_field($_GET['vector_id']) : '';
5665		-
5666		- if (empty($vector_id)) {
5667		- set_transient('mxchat_admin_notice_error',
5668		- esc_html__('Invalid vector ID.', 'mxchat'),
5669		- 30
5670		- );
5671		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
5672		- exit;
5673		- }
5674		-
5675		- // Get Pinecone settings
	3318	+ // Check if Pinecone is enabled
5676	3319	$pinecone_options = get_option('mxchat_pinecone_addon_options', array());
5677	3320	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5678		-
5679		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
5680		- set_transient('mxchat_admin_notice_error',
5681		- esc_html__('Pinecone is not properly configured.', 'mxchat'),
5682		- 30
5683		- );
5684		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
5685		- exit;
5686		- }
5687		-
5688		- // Delete from Pinecone
5689		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
5690		- $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
5691		- $vector_id,
5692		- $pinecone_options['mxchat_pinecone_api_key'],
5693		- $pinecone_options['mxchat_pinecone_host']
5694		- );
5695		-
5696		- if ($result['success']) {
5697		- // No cache clearing needed since we removed caching
5698		- set_transient('mxchat_admin_notice_success',
5699		- esc_html__('Entry deleted successfully from Pinecone.', 'mxchat'),
5700		- 30
5701		- );
5702		- } else {
5703		- set_transient('mxchat_admin_notice_error',
5704		- esc_html__('Failed to delete entry: ', 'mxchat') . $result['message'],
5705		- 30
5706		- );
5707		- }
5708		-
5709		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
5710		- exit;
5711		-}
5712		-/**
5713		- * Handle individual Pinecone content deletion via AJAX
5714		- */
5715		-public function ajax_mxchat_delete_pinecone_prompt() {
5716		- // Verify nonce and permissions
5717		- if (!check_ajax_referer('mxchat_delete_pinecone_prompt_nonce', 'nonce', false)) {
5718		- wp_send_json_error('Invalid nonce');
5719		- exit;
5720		- }
5721		-
5722		- if (!current_user_can('manage_options')) {
5723		- wp_send_json_error('Unauthorized access');
5724		- exit;
5725		- }
5726		-
5727		- $vector_id = isset($_POST['vector_id']) ? sanitize_text_field($_POST['vector_id']) : '';
5728		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
5729		-
5730		- if (empty($vector_id)) {
5731		- wp_send_json_error('Missing vector ID');
5732		- exit;
5733		- }
5734		-
5735		- // Get bot-specific Pinecone settings
5736		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
5737		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
5738		-
5739		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5740		-
5741		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
5742		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
5743		- exit;
5744		- }
5745		-
5746		- // Delete from the correct Pinecone index
5747		- $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
5748		- $vector_id,
5749		- $pinecone_options['mxchat_pinecone_api_key'],
5750		- $pinecone_options['mxchat_pinecone_host']
5751		- );
5752		-
5753		- if ($result['success']) {
5754		- // No cache clearing needed since we removed caching
5755		- wp_send_json_success(array(
5756		- 'message' => 'Entry deleted successfully from Pinecone',
5757		- 'vector_id' => $vector_id,
5758		- 'bot_id' => $bot_id
5759		- ));
5760		- } else {
5761		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete from Pinecone: ' . $result['message'], array('vector_id' => $vector_id, 'bot_id' => $bot_id));
5762		- wp_send_json_error('Failed to delete from Pinecone: ' . $result['message']);
5763		- }
5764		-
5765		- exit;
5766		-}
5767	3321
5768		-/**
5769		- * Handle deletion of all chunks for a given source URL via AJAX
5770		- * Follows the same pattern as ajax_mxchat_delete_pinecone_prompt
5771		- */
5772		-public function ajax_mxchat_delete_chunks_by_url() {
5773		- // Verify nonce and permissions
5774		- if (!check_ajax_referer('mxchat_delete_chunks_nonce', 'nonce', false)) {
5775		- wp_send_json_error('Invalid nonce');
5776		- exit;
5777		- }
5778		-
5779		- if (!current_user_can('manage_options')) {
5780		- wp_send_json_error('Unauthorized access');
5781		- exit;
5782		- }
5783		-
5784		- $source_url = isset($_POST['source_url']) ? esc_url_raw($_POST['source_url']) : '';
5785		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
5786		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
5787		-
5788		- if (empty($source_url)) {
5789		- wp_send_json_error('Missing source URL');
5790		- exit;
5791		- }
5792		-
5793		- // Generate the base vector ID from the source URL (same as how chunks are created)
5794		- $base_vector_id = md5($source_url);
5795		-
5796		- if ($data_source === 'pinecone') {
5797		- // Get bot-specific Pinecone settings (same as working delete function)
5798		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
5799		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
5800		-
5801		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5802		-
5803		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
5804		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
5805		- exit;
5806		- }
5807		-
5808		- $api_key = $pinecone_options['mxchat_pinecone_api_key'];
5809		- $host = $pinecone_options['mxchat_pinecone_host'];
5810		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
5811		-
5812		- // Collect all vector IDs to delete
5813		- $vectors_to_delete = array();
5814		-
5815		- // Add the original single-vector ID (for non-chunked content)
5816		- $vectors_to_delete[] = $base_vector_id;
5817		-
5818		- // Use Pinecone list API to find all chunk vectors with this prefix
5819		- // NOTE: Pinecone List API is a GET request with query parameters, not POST
5820		- $prefix = $base_vector_id . '_chunk_';
5821		-
5822		- $query_params = array(
5823		- 'prefix' => $prefix,
5824		- 'limit' => 100
5825		- );
5826		-
5827		- if (!empty($namespace)) {
5828		- $query_params['namespace'] = $namespace;
5829		- }
5830		-
5831		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
5832		-
5833		- $list_response = wp_remote_get($list_url, array(
5834		- 'headers' => array(
5835		- 'Api-Key' => $api_key,
5836		- 'accept' => 'application/json'
5837		- ),
5838		- 'timeout' => 30
5839		- ));
5840		-
5841		- if (!is_wp_error($list_response)) {
5842		- $list_body_response = wp_remote_retrieve_body($list_response);
5843		- $list_data = json_decode($list_body_response, true);
5844		- if (!empty($list_data['vectors'])) {
5845		- foreach ($list_data['vectors'] as $vector) {
5846		- if (isset($vector['id'])) {
5847		- $vectors_to_delete[] = $vector['id'];
5848		- }
5849		- }
5850		- }
5851		- }
5852		-
5853		- if (empty($vectors_to_delete)) {
5854		- wp_send_json_success(array(
5855		- 'message' => 'No vectors found to delete',
5856		- 'source_url' => $source_url
5857		- ));
5858		- exit;
5859		- }
5860		-
5861		- // Delete all vectors using the same endpoint as the working function
5862		- $delete_url = "https://{$host}/vectors/delete";
5863		-
5864		- $delete_body = array(
5865		- 'ids' => $vectors_to_delete
5866		- );
5867		-
5868		- if (!empty($namespace)) {
5869		- $delete_body['namespace'] = $namespace;
5870		- }
5871		-
5872		- $delete_response = wp_remote_post($delete_url, array(
5873		- 'headers' => array(
5874		- 'Api-Key' => $api_key,
5875		- 'accept' => 'application/json',
5876		- 'content-type' => 'application/json'
5877		- ),
5878		- 'body' => wp_json_encode($delete_body),
5879		- 'timeout' => 30
5880		- ));
5881		-
5882		- if (is_wp_error($delete_response)) {
5883		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete chunks from Pinecone: ' . $delete_response->get_error_message(), array('source_url' => $source_url));
5884		- wp_send_json_error('Failed to delete from Pinecone: ' . $delete_response->get_error_message());
5885		- exit;
5886		- }
5887		-
5888		- $response_code = wp_remote_retrieve_response_code($delete_response);
5889		-
5890		- if ($response_code !== 200) {
5891		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Pinecone API error (HTTP ' . $response_code . ')', array('source_url' => $source_url));
5892		- wp_send_json_error('Pinecone API error (HTTP ' . $response_code . ')');
5893		- exit;
5894		- }
5895		-
5896		- wp_send_json_success(array(
5897		- 'message' => 'All chunks deleted successfully from Pinecone',
5898		- 'source_url' => $source_url,
5899		- 'deleted_count' => count($vectors_to_delete)
5900		- ));
5901		-
	3322	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3323	+ // Delete from Pinecone
	3324	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
5902	3325	} else {
5903		- // WordPress database deletion
	3326	+ // Delete from WordPress DB
5904	3327	global $wpdb;
5905	3328	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
5906		-
5907		- $result = $wpdb->delete(
5908		- $table_name,
5909		- array('source_url' => $source_url),
5910		- array('%s')
5911		- );
5912		-
5913		- if ($result === false) {
5914		- MxChat_Admin::mxchat_log_debug('database_error', 'Failed to delete from database: ' . $wpdb->last_error, array('source_url' => $source_url));
5915		- wp_send_json_error('Failed to delete from database: ' . $wpdb->last_error);
5916		- exit;
5917		- }
5918		-
5919		- wp_send_json_success(array(
5920		- 'message' => 'All chunks deleted successfully from database',
5921		- 'source_url' => $source_url,
5922		- 'deleted_count' => $result
5923		- ));
5924		- }
5925		-
5926		- exit;
5927		-}
5928		-
5929		-/**
5930		- * Handle individual WordPress database content deletion via AJAX
5931		- * Mirrors the Pinecone delete handler but for WordPress database entries
5932		- */
5933		-public function ajax_mxchat_delete_wordpress_prompt() {
5934		- // Verify nonce and permissions
5935		- if (!check_ajax_referer('mxchat_delete_wordpress_prompt_nonce', 'nonce', false)) {
5936		- wp_send_json_error('Invalid nonce');
5937		- exit;
5938		- }
5939		-
5940		- if (!current_user_can('manage_options')) {
5941		- wp_send_json_error('Unauthorized access');
5942		- exit;
5943		- }
5944		-
5945		- $entry_id = isset($_POST['entry_id']) ? intval($_POST['entry_id']) : 0;
5946		-
5947		- if (empty($entry_id)) {
5948		- wp_send_json_error('Missing entry ID');
5949		- exit;
5950		- }
5951		-
5952		- global $wpdb;
5953		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
5954		-
5955		- // Clear cache for this entry
5956		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
5957		-
5958		- // Delete from database
5959		- $result = $wpdb->delete(
5960		- $table_name,
5961		- array('id' => $entry_id),
5962		- array('%d')
5963		- );
5964		-
5965		- if ($result !== false) {
5966		- wp_send_json_success(array(
5967		- 'message' => 'Entry deleted successfully',
5968		- 'entry_id' => $entry_id
5969		- ));
5970		- } else {
5971		- wp_send_json_error('Failed to delete entry from database');
5972		- }
5973		-
5974		- exit;
5975		-}
5976		-
5977		-/**
5978		- * Handle bulk deletion of knowledge entries via AJAX
5979		- * Supports both Pinecone and WordPress database entries
5980		- */
5981		-public function ajax_mxchat_bulk_delete_knowledge() {
5982		- // Verify nonce and permissions
5983		- if (!check_ajax_referer('mxchat_bulk_delete_knowledge_nonce', 'nonce', false)) {
5984		- wp_send_json_error('Invalid nonce');
5985		- exit;
5986		- }
5987		-
5988		- if (!current_user_can('manage_options')) {
5989		- wp_send_json_error('Unauthorized access');
5990		- exit;
5991		- }
5992		-
5993		- $entries = isset($_POST['entries']) ? $_POST['entries'] : array();
5994		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
5995		-
5996		- if (empty($entries) || !is_array($entries)) {
5997		- wp_send_json_error('No entries provided');
5998		- exit;
5999		- }
6000		-
6001		- // Extend execution time — bulk Pinecone operations can take a while
6002		- if (function_exists('set_time_limit')) {
6003		- set_time_limit(120);
6004		- }
6005		-
6006		- $success_ids = array();
6007		- $failed_ids = array();
6008		- $errors = array();
6009		-
6010		- global $wpdb;
6011		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6012		-
6013		- // Get Pinecone manager for Pinecone deletions
6014		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6015		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6016		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6017		-
6018		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
6019		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
6020		-
6021		- // =============================================
6022		- // PHASE 1: Collect all Pinecone vector IDs
6023		- // and separate WordPress entries
6024		- // =============================================
6025		- $pinecone_entry_ids = array(); // entry IDs that are pinecone-sourced
6026		- $wordpress_entries = array(); // entries for WordPress DB deletion
6027		- $all_vector_ids = array(); // all pinecone vector IDs to delete in one batch
6028		-
6029		- foreach ($entries as $entry) {
6030		- $entry_id = sanitize_text_field($entry['id'] ?? '');
6031		- $source = sanitize_text_field($entry['source'] ?? 'wordpress');
6032		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
6033		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
6034		-
6035		- if (empty($entry_id)) {
6036		- continue;
6037		- }
6038		-
6039		- if ($source === 'pinecone') {
6040		- if (!$use_pinecone || empty($api_key)) {
6041		- $failed_ids[] = $entry_id;
6042		- $errors[] = "Pinecone not configured for entry: $entry_id";
6043		- continue;
6044		- }
6045		-
6046		- $pinecone_entry_ids[] = $entry_id;
6047		-
6048		- if ($is_group && !empty($source_url)) {
6049		- // Grouped/chunked entry: collect base ID + chunk IDs via List API
6050		- $base_vector_id = md5($source_url);
6051		- $all_vector_ids[] = $base_vector_id;
6052		-
6053		- $list_url = 'https://' . $host . '/vectors/list?prefix=' . urlencode($base_vector_id . '_chunk_') . '&limit=100';
6054		- $list_response = wp_remote_get($list_url, array(
6055		- 'headers' => array(
6056		- 'Api-Key' => $api_key,
6057		- 'accept' => 'application/json'
6058		- ),
6059		- 'timeout' => 30
6060		- ));
6061		-
6062		- if (!is_wp_error($list_response)) {
6063		- $list_body = json_decode(wp_remote_retrieve_body($list_response), true);
6064		- if (!empty($list_body['vectors']) && is_array($list_body['vectors'])) {
6065		- foreach ($list_body['vectors'] as $vector) {
6066		- if (isset($vector['id'])) {
6067		- $all_vector_ids[] = $vector['id'];
6068		- }
6069		- }
6070		- }
6071		- }
6072		- } else {
6073		- // Single entry: the entry_id IS the vector ID
6074		- $all_vector_ids[] = $entry_id;
6075		- }
6076		- } else {
6077		- $wordpress_entries[] = $entry;
6078		- }
6079		- }
6080		-
6081		- // =============================================
6082		- // PHASE 2: Single batch delete to Pinecone
6083		- // =============================================
6084		- if (!empty($all_vector_ids)) {
6085		- $all_vector_ids = array_values(array_unique($all_vector_ids));
6086		- $pinecone_success = true;
6087		- $batches = array_chunk($all_vector_ids, 100);
6088		-
6089		- foreach ($batches as $batch) {
6090		- $delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
6091		- 'headers' => array(
6092		- 'Api-Key' => $api_key,
6093		- 'accept' => 'application/json',
6094		- 'content-type' => 'application/json'
6095		- ),
6096		- 'body' => wp_json_encode(array('ids' => $batch)),
6097		- 'timeout' => 60
6098		- ));
6099		-
6100		- if (is_wp_error($delete_response)) {
6101		- $pinecone_success = false;
6102		- $errors[] = 'Pinecone batch deletion failed: ' . $delete_response->get_error_message();
6103		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed: ' . $delete_response->get_error_message());
6104		- } else {
6105		- $response_code = wp_remote_retrieve_response_code($delete_response);
6106		- if ($response_code !== 200) {
6107		- $pinecone_success = false;
6108		- $response_body = wp_remote_retrieve_body($delete_response);
6109		- $errors[] = "Pinecone API error (HTTP $response_code)";
6110		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed (HTTP ' . $response_code . ')', array('response' => substr($response_body, 0, 200)));
6111		- }
6112		- }
6113		- }
6114		-
6115		- // Mark all pinecone entries based on batch result
6116		- foreach ($pinecone_entry_ids as $eid) {
6117		- if ($pinecone_success) {
6118		- $success_ids[] = $eid;
6119		- } else {
6120		- $failed_ids[] = $eid;
6121		- }
6122		- }
6123		- }
6124		-
6125		- // =============================================
6126		- // PHASE 3: WordPress database deletions
6127		- // =============================================
6128		- foreach ($wordpress_entries as $entry) {
6129		- $entry_id = sanitize_text_field($entry['id'] ?? '');
6130		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
6131		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
6132		-
6133		- if (empty($entry_id)) {
6134		- continue;
6135		- }
6136		-
6137		- try {
6138		- if ($is_group && !empty($source_url)) {
6139		- $result = $wpdb->delete(
6140		- $table_name,
6141		- array('source_url' => $source_url),
6142		- array('%s')
6143		- );
6144		- } else {
6145		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
6146		- $result = $wpdb->delete(
6147		- $table_name,
6148		- array('id' => intval($entry_id)),
6149		- array('%d')
6150		- );
6151		- }
6152		-
6153		- if ($result !== false) {
6154		- $success_ids[] = $entry_id;
6155		- } else {
6156		- $failed_ids[] = $entry_id;
6157		- $errors[] = "Database error for entry: $entry_id";
6158		- }
6159		- } catch (Exception $e) {
6160		- $failed_ids[] = $entry_id;
6161		- $errors[] = $e->getMessage();
6162		- }
6163		- }
6164		-
6165		- wp_send_json_success(array(
6166		- 'success_ids' => $success_ids,
6167		- 'failed_ids' => $failed_ids,
6168		- 'errors' => $errors,
6169		- 'total_processed' => count($success_ids) + count($failed_ids)
6170		- ));
6171		-
6172		- exit;
6173		-}
6174		-
6175		-/**
6176		- * Get hierarchical roles for dropdown
6177		- */
6178		-public function mxchat_get_role_options() {
6179		- return array(
6180		- 'public' => __('Public (Everyone)', 'mxchat'),
6181		- 'logged_in' => __('Logged In Users', 'mxchat'),
6182		- 'subscriber' => __('Subscribers & Above', 'mxchat'),
6183		- 'contributor' => __('Contributors & Above', 'mxchat'),
6184		- 'author' => __('Authors & Above', 'mxchat'),
6185		- 'editor' => __('Editors & Above', 'mxchat'),
6186		- 'administrator' => __('Administrators Only', 'mxchat')
6187		- );
6188		-}
6189		-
6190		-/**
6191		- * Check if user has access to content based on role restriction
6192		- */
6193		-public function mxchat_user_has_content_access($role_restriction) {
6194		- // Public content is always accessible
6195		- if ($role_restriction === 'public' || empty($role_restriction)) {
6196		- return true;
6197		- }
6198		-
6199		- // Check if user is logged in for logged_in restriction
6200		- if ($role_restriction === 'logged_in') {
6201		- return is_user_logged_in();
6202		- }
6203		-
6204		- // If not logged in, no access to role-restricted content
6205		- if (!is_user_logged_in()) {
6206		- return false;
6207		- }
6208		-
6209		- $user = wp_get_current_user();
6210		- $user_roles = $user->roles;
6211		-
6212		- if (empty($user_roles)) {
6213		- return false;
6214		- }
6215		-
6216		- // Define role hierarchy (higher number = higher access)
6217		- $hierarchy = array(
6218		- 'subscriber' => 1,
6219		- 'contributor' => 2,
6220		- 'author' => 3,
6221		- 'editor' => 4,
6222		- 'administrator' => 5
6223		- );
6224		-
6225		- // Get required level
6226		- $required_level = isset($hierarchy[$role_restriction]) ? $hierarchy[$role_restriction] : 0;
6227		-
6228		- // Check if user has required level or higher
6229		- foreach ($user_roles as $user_role) {
6230		- $user_level = isset($hierarchy[$user_role]) ? $hierarchy[$user_role] : 0;
6231		- if ($user_level >= $required_level) {
6232		- return true;
6233		- }
6234		- }
6235		-
6236		- return false;
6237		-}
6238		-
6239		-/**
6240		- * Handle role restriction updates via AJAX
6241		- * Removed cache clearing call since we removed caching
6242		- */
6243		-public function ajax_mxchat_update_role_restriction() {
6244		- // Verify nonce and permissions
6245		- if (!check_ajax_referer('mxchat_update_role_nonce', 'nonce', false)) {
6246		- wp_send_json_error('Invalid nonce');
6247		- exit;
6248		- }
6249		-
6250		- if (!current_user_can('manage_options')) {
6251		- wp_send_json_error('Unauthorized access');
6252		- exit;
6253		- }
6254		-
6255		- $entry_id = isset($_POST['entry_id']) ? sanitize_text_field($_POST['entry_id']) : '';
6256		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
6257		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
6258		-
6259		- if (empty($entry_id)) {
6260		- wp_send_json_error('Invalid entry ID');
6261		- exit;
6262		- }
6263		-
6264		- // Get knowledge manager instance to validate role restriction
6265		- $knowledge_manager = MxChat_Knowledge_Manager::get_instance();
6266		- $valid_roles = array_keys($knowledge_manager->mxchat_get_role_options());
6267		- if (!in_array($role_restriction, $valid_roles)) {
6268		- wp_send_json_error('Invalid role restriction');
6269		- exit;
6270		- }
6271		-
6272		- global $wpdb;
6273		-
6274		- if ($data_source === 'pinecone') {
6275		- // Handle Pinecone role restriction (stored separately in WordPress table)
6276		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6277	3329
6278		- // Use REPLACE to insert or update the role restriction
6279		- $result = $wpdb->replace(
6280		- $roles_table,
6281		- array(
6282		- 'vector_id' => $entry_id,
6283		- 'role_restriction' => $role_restriction,
6284		- 'updated_at' => current_time('mysql')
6285		- ),
6286		- array('%s', '%s', '%s')
6287		- );
6288		-
6289		- // No cache clearing needed since we removed caching
6290		-
6291		- } else {
6292		- // Handle WordPress database role restriction (existing functionality)
6293		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6294		-
6295		- $result = $wpdb->update(
	3330	+ $wpdb->delete(
6296	3331	$table_name,
6297		- array('role_restriction' => $role_restriction),
6298		- array('id' => absint($entry_id)),
6299		- array('%s'),
6300		- array('%d')
6301		- );
6302		- }
6303		-
6304		- if ($result === false) {
6305		- wp_send_json_error('Database update failed: ' . $wpdb->last_error);
6306		- exit;
6307		- }
6308		-
6309		- wp_send_json_success(array(
6310		- 'message' => 'Role restriction updated successfully',
6311		- 'role_restriction' => $role_restriction,
6312		- 'data_source' => $data_source,
6313		- 'entry_id' => $entry_id
6314		- ));
6315		- exit;
6316		-}
6317		-
6318		-// ========================================
6319		-// ROLE-BASED CONTENT RESTRICTIONS - NEW FUNCTIONS
6320		-// Add these to your MxChat_Knowledge_Manager class
6321		-// ========================================
6322		-
6323		-/**
6324		- * Initialize role-based content hooks
6325		- * Add this call to your __construct() or mxchat_init_hooks() method
6326		- */
6327		-private function mxchat_init_role_hooks() {
6328		- // AJAX handlers for tag-role mappings
6329		- add_action('wp_ajax_mxchat_add_tag_role_mapping', array($this, 'ajax_add_tag_role_mapping'));
6330		- add_action('wp_ajax_mxchat_delete_tag_role_mapping', array($this, 'ajax_delete_tag_role_mapping'));
6331		- add_action('wp_ajax_mxchat_get_tag_role_mappings', array($this, 'ajax_get_tag_role_mappings'));
6332		- add_action('wp_ajax_mxchat_bulk_update_tag_roles', array($this, 'ajax_bulk_update_tag_roles'));
6333		-
6334		- // Hook to automatically update role restrictions when tags are added/removed
6335		- add_action('set_object_terms', array($this, 'handle_tag_change'), 10, 6);
6336		-
6337		- // Hook to apply role restrictions on auto-sync
6338		- add_action('mxchat_content_stored', array($this, 'apply_role_restriction_after_storage'), 10, 2);
6339		-}
6340		-
6341		-/**
6342		- * Add tag-role mapping via AJAX
6343		- */
6344		-public function ajax_add_tag_role_mapping() {
6345		- // Verify nonce and permissions
6346		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6347		-
6348		- if (!current_user_can('manage_options')) {
6349		- wp_send_json_error('Unauthorized access');
6350		- exit;
6351		- }
6352		-
6353		- $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
6354		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
6355		-
6356		- if (empty($tag_slug)) {
6357		- wp_send_json_error('Tag slug is required');
6358		- exit;
6359		- }
6360		-
6361		- // Validate role restriction
6362		- $valid_roles = array_keys($this->mxchat_get_role_options());
6363		- if (!in_array($role_restriction, $valid_roles)) {
6364		- wp_send_json_error('Invalid role restriction');
6365		- exit;
6366		- }
6367		-
6368		- // Check if tag exists in WordPress
6369		- $term = get_term_by('slug', $tag_slug, 'post_tag');
6370		- if (!$term) {
6371		- wp_send_json_error('Tag does not exist in WordPress');
6372		- exit;
6373		- }
6374		-
6375		- // Get existing mappings
6376		- $mappings = get_option('mxchat_tag_role_mappings', array());
6377		-
6378		- // Check if mapping already exists
6379		- if (isset($mappings[$tag_slug])) {
6380		- wp_send_json_error('Mapping for this tag already exists');
6381		- exit;
6382		- }
6383		-
6384		- // Add new mapping
6385		- $mappings[$tag_slug] = $role_restriction;
6386		- update_option('mxchat_tag_role_mappings', $mappings);
6387		-
6388		- wp_send_json_success(array(
6389		- 'message' => 'Tag-role mapping added successfully',
6390		- 'tag_slug' => $tag_slug,
6391		- 'role_restriction' => $role_restriction
6392		- ));
6393		- exit;
6394		-}
6395		-
6396		-/**
6397		- * Delete tag-role mapping via AJAX
6398		- */
6399		-public function ajax_delete_tag_role_mapping() {
6400		- // Verify nonce and permissions
6401		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6402		-
6403		- if (!current_user_can('manage_options')) {
6404		- wp_send_json_error('Unauthorized access');
6405		- exit;
6406		- }
6407		-
6408		- $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
6409		-
6410		- if (empty($tag_slug)) {
6411		- wp_send_json_error('Tag slug is required');
6412		- exit;
6413		- }
6414		-
6415		- // Get existing mappings
6416		- $mappings = get_option('mxchat_tag_role_mappings', array());
6417		-
6418		- // Check if mapping exists
6419		- if (!isset($mappings[$tag_slug])) {
6420		- wp_send_json_error('Mapping does not exist');
6421		- exit;
6422		- }
6423		-
6424		- // Remove mapping
6425		- unset($mappings[$tag_slug]);
6426		- update_option('mxchat_tag_role_mappings', $mappings);
6427		-
6428		- wp_send_json_success(array(
6429		- 'message' => 'Tag-role mapping deleted successfully',
6430		- 'tag_slug' => $tag_slug
6431		- ));
6432		- exit;
6433		-}
6434		-
6435		-/**
6436		- * Get all tag-role mappings via AJAX
6437		- */
6438		-public function ajax_get_tag_role_mappings() {
6439		- // Verify nonce and permissions
6440		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6441		-
6442		- if (!current_user_can('manage_options')) {
6443		- wp_send_json_error('Unauthorized access');
6444		- exit;
6445		- }
6446		-
6447		- // Get mappings
6448		- $mappings = get_option('mxchat_tag_role_mappings', array());
6449		- $role_options = $this->mxchat_get_role_options();
6450		-
6451		- $formatted_mappings = array();
6452		-
6453		- foreach ($mappings as $tag_slug => $role_restriction) {
6454		- // Get tag object
6455		- $term = get_term_by('slug', $tag_slug, 'post_tag');
6456		-
6457		- // Count posts with this tag
6458		- $post_count = 0;
6459		- if ($term) {
6460		- $post_count = $term->count;
6461		- }
6462		-
6463		- $formatted_mappings[] = array(
6464		- 'tag_slug' => $tag_slug,
6465		- 'role_restriction' => $role_restriction,
6466		- 'role_label' => $role_options[$role_restriction] ?? $role_restriction,
6467		- 'post_count' => $post_count
6468		- );
6469		- }
6470		-
6471		- wp_send_json_success(array(
6472		- 'mappings' => $formatted_mappings
6473		- ));
6474		- exit;
6475		-}
6476		-
6477		-/**
6478		- * Bulk update role restrictions for all existing content with mapped tags
6479		- */
6480		-public function ajax_bulk_update_tag_roles() {
6481		- // Verify nonce and permissions
6482		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6483		-
6484		- if (!current_user_can('manage_options')) {
6485		- wp_send_json_error('Unauthorized access');
6486		- exit;
6487		- }
6488		-
6489		- // Get mappings
6490		- $mappings = get_option('mxchat_tag_role_mappings', array());
6491		-
6492		- if (empty($mappings)) {
6493		- wp_send_json_error('No tag-role mappings found');
6494		- exit;
6495		- }
6496		-
6497		- global $wpdb;
6498		-
6499		- // Check if using Pinecone
6500		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6501		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6502		-
6503		- $updated_count = 0;
6504		- $details = array();
6505		-
6506		- foreach ($mappings as $tag_slug => $role_restriction) {
6507		- // Get all posts with this tag
6508		- $posts = get_posts(array(
6509		- 'tag' => $tag_slug,
6510		- 'post_type' => 'any',
6511		- 'posts_per_page' => -1,
6512		- 'fields' => 'ids',
6513		- 'post_status' => 'publish'
6514		- ));
6515		-
6516		- if (empty($posts)) {
6517		- continue;
6518		- }
6519		-
6520		- $tag_updated = 0;
6521		-
6522		- foreach ($posts as $post_id) {
6523		- $source_url = get_permalink($post_id);
6524		- if (!$source_url) {
6525		- continue;
6526		- }
6527		-
6528		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
6529		- // Update Pinecone role restriction
6530		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6531		- $vector_id = md5($source_url);
6532		-
6533		- $result = $wpdb->replace(
6534		- $roles_table,
6535		- array(
6536		- 'vector_id' => $vector_id,
6537		- 'role_restriction' => $role_restriction,
6538		- 'updated_at' => current_time('mysql')
6539		- ),
6540		- array('%s', '%s', '%s')
6541		- );
6542		- } else {
6543		- // Update WordPress DB
6544		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6545		-
6546		- $result = $wpdb->update(
6547		- $table_name,
6548		- array('role_restriction' => $role_restriction),
6549		- array('source_url' => $source_url),
6550		- array('%s'),
6551		- array('%s')
6552		- );
6553		- }
6554		-
6555		- if ($result !== false) {
6556		- $tag_updated++;
6557		- $updated_count++;
6558		- }
6559		- }
6560		-
6561		- if ($tag_updated > 0) {
6562		- $details[] = sprintf(
6563		- 'Tag "%s" (%s): %d posts updated',
6564		- $tag_slug,
6565		- $role_restriction,
6566		- $tag_updated
6567		- );
6568		- }
6569		- }
6570		-
6571		- wp_send_json_success(array(
6572		- 'message' => 'Bulk update completed',
6573		- 'updated_count' => $updated_count,
6574		- 'tags_processed' => count($mappings),
6575		- 'details' => $details
6576		- ));
6577		- exit;
6578		-}
6579		-
6580		-/**
6581		- * Handle tag changes on posts (when tags are added or removed)
6582		- */
6583		-public function handle_tag_change($object_id, $terms, $tt_ids, $taxonomy, $append, $old_tt_ids) {
6584		- // Only process post tags
6585		- if ($taxonomy !== 'post_tag') {
6586		- return;
6587		- }
6588		-
6589		- // Get tag-role mappings
6590		- $mappings = get_option('mxchat_tag_role_mappings', array());
6591		-
6592		- if (empty($mappings)) {
6593		- return;
6594		- }
6595		-
6596		- // Get the post's URL
6597		- $source_url = get_permalink($object_id);
6598		- if (!$source_url) {
6599		- return;
6600		- }
6601		-
6602		- // Determine the highest role restriction based on tags
6603		- $highest_role = 'public';
6604		- $role_hierarchy = array(
6605		- 'public' => 0,
6606		- 'logged_in' => 1,
6607		- 'subscriber' => 2,
6608		- 'contributor' => 3,
6609		- 'author' => 4,
6610		- 'editor' => 5,
6611		- 'administrator' => 6
6612		- );
6613		-
6614		- // Get all current tags for the post
6615		- $current_tags = wp_get_post_tags($object_id, array('fields' => 'slugs'));
6616		-
6617		- // Find the highest role restriction among the tags
6618		- foreach ($current_tags as $tag_slug) {
6619		- if (isset($mappings[$tag_slug])) {
6620		- $role = $mappings[$tag_slug];
6621		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
6622		- $highest_role = $role;
6623		- }
6624		- }
6625		- }
6626		-
6627		- // Update the role restriction in the database
6628		- global $wpdb;
6629		-
6630		- // Check if using Pinecone
6631		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6632		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6633		-
6634		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
6635		- // Update Pinecone role restriction
6636		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6637		- $vector_id = md5($source_url);
6638		-
6639		- $wpdb->replace(
6640		- $roles_table,
6641		- array(
6642		- 'vector_id' => $vector_id,
6643		- 'role_restriction' => $highest_role,
6644		- 'updated_at' => current_time('mysql')
6645		- ),
6646		- array('%s', '%s', '%s')
6647		- );
6648		- } else {
6649		- // Update WordPress DB
6650		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6651		-
6652		- $wpdb->update(
6653		- $table_name,
6654		- array('role_restriction' => $highest_role),
6655	3332	array('source_url' => $source_url),
6656		- array('%s'),
6657	3333	array('%s')
6658	3334	);
6659	3335	}
6660	3336	}
6661		-
6662		-/**
6663		- * Apply role restriction after content is stored (for auto-sync)
6664		- */
6665		-public function apply_role_restriction_after_storage($post_id, $source_url) {
6666		- // Get tag-role mappings
6667		- $mappings = get_option('mxchat_tag_role_mappings', array());
6668	3337
6669		- if (empty($mappings)) {
6670		- return;
6671		- }
6672		-
6673		- // Get all tags for the post
6674		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
6675		-
6676		- if (empty($post_tags)) {
6677		- return;
6678		- }
6679		-
6680		- // Determine the highest role restriction based on tags
6681		- $highest_role = 'public';
6682		- $role_hierarchy = array(
6683		- 'public' => 0,
6684		- 'logged_in' => 1,
6685		- 'subscriber' => 2,
6686		- 'contributor' => 3,
6687		- 'author' => 4,
6688		- 'editor' => 5,
6689		- 'administrator' => 6
6690		- );
6691		-
6692		- foreach ($post_tags as $tag_slug) {
6693		- if (isset($mappings[$tag_slug])) {
6694		- $role = $mappings[$tag_slug];
6695		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
6696		- $highest_role = $role;
6697		- }
6698		- }
6699		- }
6700		-
6701		- // If no restricted tags found, return (leave as public)
6702		- if ($highest_role === 'public') {
6703		- return;
6704		- }
6705		-
6706		- // Update the role restriction
6707		- global $wpdb;
6708		-
6709		- // Check if using Pinecone
6710		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6711		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6712		-
6713		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
6714		- // Update Pinecone role restriction
6715		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6716		- $vector_id = md5($source_url);
6717		-
6718		- $wpdb->replace(
6719		- $roles_table,
6720		- array(
6721		- 'vector_id' => $vector_id,
6722		- 'role_restriction' => $highest_role,
6723		- 'updated_at' => current_time('mysql')
6724		- ),
6725		- array('%s', '%s', '%s')
6726		- );
6727		- } else {
6728		- // Update WordPress DB
6729		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6730		-
6731		- $wpdb->update(
6732		- $table_name,
6733		- array('role_restriction' => $highest_role),
6734		- array('source_url' => $source_url),
6735		- array('%s'),
6736		- array('%s')
6737		- );
6738		- }
6739		-}
6740		-
6741		-
6742	3338	// ========================================
6743	3339	// HELPER METHODS
6744	3340	// ========================================
6745	3341
		@@ -6806,1074 +3402,8 @@
6806	3402	*/
6807	3403	private function mxchat_get_pinecone_manager() {
6808	3404	return MxChat_Pinecone_Manager::get_instance();
6809	3405	}
6810		-
6811		-
6812		- // ========================================
6813		-// DATABASE QUEUE TABLE MANAGEMENT
6814		-// ========================================
6815		-
6816		-/**
6817		- * Create queue table on plugin activation
6818		- * Call this from your plugin activation hook
6819		- */
6820		-public function mxchat_create_queue_table() {
6821		- global $wpdb;
6822		-
6823		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
6824		- $charset_collate = $wpdb->get_charset_collate();
6825		-
6826		- $sql = "CREATE TABLE IF NOT EXISTS $table_name (
6827		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
6828		- queue_id varchar(64) NOT NULL,
6829		- item_type varchar(20) NOT NULL,
6830		- item_data longtext NOT NULL,
6831		- status varchar(20) NOT NULL DEFAULT 'pending',
6832		- bot_id varchar(50) NOT NULL DEFAULT 'default',
6833		- priority int(11) NOT NULL DEFAULT 0,
6834		- attempts int(11) NOT NULL DEFAULT 0,
6835		- max_attempts int(11) NOT NULL DEFAULT 3,
6836		- error_message text DEFAULT NULL,
6837		- created_at datetime NOT NULL,
6838		- started_at datetime DEFAULT NULL,
6839		- completed_at datetime DEFAULT NULL,
6840		- PRIMARY KEY (id),
6841		- KEY queue_id (queue_id),
6842		- KEY status (status),
6843		- KEY item_type (item_type),
6844		- KEY priority (priority)
6845		- ) $charset_collate;";
6846		-
6847		- require_once(ABSPATH . 'wp-admin/includes/upgrade.php');
6848		- dbDelta($sql);
6849		-
6850		- // Also create a meta table for queue metadata
6851		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
6852		-
6853		- $meta_sql = "CREATE TABLE IF NOT EXISTS $meta_table (
6854		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
6855		- queue_id varchar(64) NOT NULL,
6856		- meta_key varchar(255) NOT NULL,
6857		- meta_value longtext,
6858		- PRIMARY KEY (id),
6859		- KEY queue_id (queue_id),
6860		- KEY meta_key (meta_key)
6861		- ) $charset_collate;";
6862		-
6863		- dbDelta($meta_sql);
6864		-}
6865		-
6866		-/**
6867		- * Add items to the processing queue
6868		- *
6869		- * @param string $queue_id Unique identifier for this queue batch
6870		- * @param string $item_type Type of item (url, pdf_page)
6871		- * @param array $items Array of items to queue
6872		- * @param string $bot_id Bot ID for processing
6873		- * @return int Number of items queued
6874		- */
6875		-private function mxchat_add_to_queue($queue_id, $item_type, $items, $bot_id = 'default') {
6876		- global $wpdb;
6877		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
6878		-
6879		- $queued_count = 0;
6880		- $priority = 0;
6881		-
6882		- foreach ($items as $item) {
6883		- $result = $wpdb->insert(
6884		- $table_name,
6885		- array(
6886		- 'queue_id' => $queue_id,
6887		- 'item_type' => $item_type,
6888		- 'item_data' => wp_json_encode($item),
6889		- 'status' => 'pending',
6890		- 'bot_id' => $bot_id,
6891		- 'priority' => $priority,
6892		- 'attempts' => 0,
6893		- 'max_attempts' => 3,
6894		- 'created_at' => current_time('mysql')
6895		- ),
6896		- array('%s', '%s', '%s', '%s', '%s', '%d', '%d', '%d', '%s')
6897		- );
6898		-
6899		- if ($result) {
6900		- $queued_count++;
6901		- }
6902		-
6903		- $priority++; // Process in order
6904		- }
6905		-
6906		- return $queued_count;
6907		-}
6908		-
6909		-/**
6910		- * Store queue metadata (total counts, source URL, etc.)
6911		- */
6912		-private function mxchat_set_queue_meta($queue_id, $meta_key, $meta_value) {
6913		- global $wpdb;
6914		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
6915		-
6916		- // Check if meta exists
6917		- $existing = $wpdb->get_var($wpdb->prepare(
6918		- "SELECT id FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
6919		- $queue_id,
6920		- $meta_key
6921		- ));
6922		-
6923		- if ($existing) {
6924		- // Update
6925		- $wpdb->update(
6926		- $meta_table,
6927		- array('meta_value' => maybe_serialize($meta_value)),
6928		- array('queue_id' => $queue_id, 'meta_key' => $meta_key),
6929		- array('%s'),
6930		- array('%s', '%s')
6931		- );
6932		- } else {
6933		- // Insert
6934		- $wpdb->insert(
6935		- $meta_table,
6936		- array(
6937		- 'queue_id' => $queue_id,
6938		- 'meta_key' => $meta_key,
6939		- 'meta_value' => maybe_serialize($meta_value)
6940		- ),
6941		- array('%s', '%s', '%s')
6942		- );
6943		- }
6944		-}
6945		-
6946		-/**
6947		- * Get queue metadata
6948		- */
6949		-private function mxchat_get_queue_meta($queue_id, $meta_key) {
6950		- global $wpdb;
6951		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
6952		-
6953		- $value = $wpdb->get_var($wpdb->prepare(
6954		- "SELECT meta_value FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
6955		- $queue_id,
6956		- $meta_key
6957		- ));
6958		-
6959		- return maybe_unserialize($value);
6960		-}
6961		-
6962		-// ========================================
6963		-// AJAX QUEUE PROCESSING HANDLERS
6964		-// ========================================
6965		-
6966		-/**
6967		- * AJAX: Get next item from queue to process
6968		- */
6969		-public function ajax_mxchat_get_next_queue_item() {
6970		- // Verify nonce and permissions
6971		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
6972		-
6973		- if (!current_user_can('manage_options')) {
6974		- wp_send_json_error('Unauthorized access');
6975		- }
6976		-
6977		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
6978		-
6979		- if (empty($queue_id)) {
6980		- wp_send_json_error('Missing queue ID');
6981		- }
6982		-
6983		- global $wpdb;
6984		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
6985		-
6986		- // Get next pending item with retry logic for failed items
6987		- $next_item = $wpdb->get_row($wpdb->prepare(
6988		- "SELECT * FROM $table_name
6989		- WHERE queue_id = %s
6990		- AND status IN ('pending', 'failed')
6991		- AND attempts < max_attempts
6992		- ORDER BY priority ASC, id ASC
6993		- LIMIT 1",
6994		- $queue_id
6995		- ));
6996		-
6997		- if (!$next_item) {
6998		- // No more items - queue complete
6999		- wp_send_json_success(array(
7000		- 'complete' => true,
7001		- 'message' => 'Queue processing complete'
7002		- ));
7003		- }
7004		-
7005		- // Mark item as processing
7006		- $wpdb->update(
7007		- $table_name,
7008		- array(
7009		- 'status' => 'processing',
7010		- 'started_at' => current_time('mysql'),
7011		- 'attempts' => $next_item->attempts + 1
7012		- ),
7013		- array('id' => $next_item->id),
7014		- array('%s', '%s', '%d'),
7015		- array('%d')
7016		- );
7017		-
7018		- wp_send_json_success(array(
7019		- 'complete' => false,
7020		- 'item' => array(
7021		- 'id' => $next_item->id,
7022		- 'type' => $next_item->item_type,
7023		- 'data' => json_decode($next_item->item_data, true),
7024		- 'bot_id' => $next_item->bot_id,
7025		- 'attempt' => $next_item->attempts + 1
7026		- )
7027		- ));
7028		-}
7029		-
7030		-/**
7031		- * AJAX: Process a single queue item
7032		- */
7033		-public function ajax_mxchat_process_queue_item() {
7034		- // Verify nonce and permissions
7035		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7036		-
7037		- if (!current_user_can('manage_options')) {
7038		- wp_send_json_error('Unauthorized access');
7039		- }
7040		-
7041		- $item_id = isset($_POST['item_id']) ? absint($_POST['item_id']) : 0;
7042		- $item_type = isset($_POST['item_type']) ? sanitize_text_field($_POST['item_type']) : '';
7043		- $item_data = isset($_POST['item_data']) ? $_POST['item_data'] : array();
7044		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
7045		-
7046		- if (empty($item_id) || empty($item_type)) {
7047		- wp_send_json_error('Missing item data');
7048		- }
7049		-
7050		- global $wpdb;
7051		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7052		-
7053		- // Process based on item type
7054		- try {
7055		- set_time_limit(60); // Give processing 60 seconds
7056		-
7057		- $result = false;
7058		- $error_message = '';
7059		-
7060		- // Read item directly from DB to get queue_id and preserve special chars in item_data
7061		- // (POST round-trip through JS mangles characters like apostrophes in URLs)
7062		- $db_item = $wpdb->get_row($wpdb->prepare(
7063		- "SELECT queue_id, item_data FROM $table_name WHERE id = %d",
7064		- $item_id
7065		- ));
7066		- $item_queue_id = $db_item ? $db_item->queue_id : '';
7067		- if ($db_item && !empty($db_item->item_data)) {
7068		- $db_data = json_decode($db_item->item_data, true);
7069		- if (is_array($db_data)) {
7070		- $item_data = $db_data;
7071		- }
7072		- }
7073		-
7074		- switch ($item_type) {
7075		- case 'url':
7076		- $result = $this->mxchat_process_queue_url($item_data, $bot_id, $item_queue_id);
7077		- break;
7078		-
7079		- case 'pdf_page':
7080		- $result = $this->mxchat_process_queue_pdf_page($item_data, $bot_id);
7081		- break;
7082		-
7083		- default:
7084		- throw new Exception('Unknown item type: ' . $item_type);
7085		- }
7086		-
7087		- if (is_wp_error($result)) {
7088		- $error_code = $result->get_error_code();
7089		- // Content errors (empty page, sanitization) are permanent — retrying won't help
7090		- $permanent_codes = array('empty_page', 'empty_after_sanitization', 'no_api_key', 'page_not_found');
7091		- if (in_array($error_code, $permanent_codes)) {
7092		- // Mark as permanently failed — set attempts = max_attempts so it won't be retried
7093		- $current_item = $wpdb->get_row($wpdb->prepare(
7094		- "SELECT max_attempts FROM $table_name WHERE id = %d", $item_id
7095		- ));
7096		- $wpdb->update(
7097		- $table_name,
7098		- array(
7099		- 'status' => 'failed',
7100		- 'error_message' => $result->get_error_message(),
7101		- 'attempts' => $current_item ? $current_item->max_attempts : 3
7102		- ),
7103		- array('id' => $item_id),
7104		- array('%s', '%s', '%d'),
7105		- array('%d')
7106		- );
7107		- wp_send_json_error(array(
7108		- 'message' => $result->get_error_message(),
7109		- 'permanent_failure' => true,
7110		- 'item_id' => $item_id
7111		- ));
7112		- return;
7113		- }
7114		- throw new Exception($result->get_error_message());
7115		- }
7116		-
7117		- if ($result === false) {
7118		- throw new Exception('Processing returned false - item may be empty or invalid');
7119		- }
7120		-
7121		- // Mark as completed
7122		- $wpdb->update(
7123		- $table_name,
7124		- array(
7125		- 'status' => 'completed',
7126		- 'completed_at' => current_time('mysql'),
7127		- 'error_message' => null
7128		- ),
7129		- array('id' => $item_id),
7130		- array('%s', '%s', '%s'),
7131		- array('%d')
7132		- );
7133		-
7134		- wp_send_json_success(array(
7135		- 'processed' => true,
7136		- 'item_id' => $item_id,
7137		- 'message' => 'Item processed successfully'
7138		- ));
7139		-
7140		- } catch (Exception $e) {
7141		- $error_message = $e->getMessage();
7142		-
7143		- // Get current attempt count
7144		- $item = $wpdb->get_row($wpdb->prepare(
7145		- "SELECT attempts, max_attempts FROM $table_name WHERE id = %d",
7146		- $item_id
7147		- ));
7148		-
7149		- // Check if we've exhausted retries
7150		- if ($item && $item->attempts >= $item->max_attempts) {
7151		- // Permanently failed
7152		- $wpdb->update(
7153		- $table_name,
7154		- array(
7155		- 'status' => 'failed',
7156		- 'error_message' => $error_message
7157		- ),
7158		- array('id' => $item_id),
7159		- array('%s', '%s'),
7160		- array('%d')
7161		- );
7162		-
7163		- wp_send_json_error(array(
7164		- 'message' => 'Item failed after maximum attempts: ' . $error_message,
7165		- 'permanent_failure' => true,
7166		- 'item_id' => $item_id
7167		- ));
7168		- } else {
7169		- // Mark for retry
7170		- $wpdb->update(
7171		- $table_name,
7172		- array(
7173		- 'status' => 'failed',
7174		- 'error_message' => $error_message
7175		- ),
7176		- array('id' => $item_id),
7177		- array('%s', '%s'),
7178		- array('%d')
7179		- );
7180		-
7181		- wp_send_json_error(array(
7182		- 'message' => 'Item processing failed, will retry: ' . $error_message,
7183		- 'can_retry' => true,
7184		- 'item_id' => $item_id,
7185		- 'attempts' => $item ? $item->attempts : 0
7186		- ));
7187		- }
7188		- }
7189		-}
7190		-
7191		-/**
7192		- * Process a URL from the queue
7193		- */
7194		-private function mxchat_process_queue_url($item_data, $bot_id = 'default', $queue_id = '') {
7195		- $url = isset($item_data['url']) ? $item_data['url'] : '';
7196		-
7197		- if (empty($url)) {
7198		- return new WP_Error('invalid_url', 'URL is empty');
7199		- }
7200		-
7201		- // Get bot-specific API key early (needed for both paths)
7202		- $bot_options = $this->get_bot_options($bot_id);
7203		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
7204		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
7205		-
7206		- if (strpos($selected_model, 'voyage') === 0) {
7207		- $api_key = $options['voyage_api_key'] ?? '';
7208		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
7209		- $api_key = $options['gemini_api_key'] ?? '';
7210		- } else {
7211		- $api_key = $options['api_key'] ?? '';
7212		- }
7213		-
7214		- if (empty($api_key)) {
7215		- return new WP_Error('no_api_key', 'API key not configured for bot: ' . $bot_id);
7216		- }
7217		-
7218		- // Check if this is a WooCommerce product URL and WooCommerce is active
7219		- $is_product_url = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
7220		- $content_type = $is_product_url ? 'product' : 'url';
7221		-
7222		- // Try to get WooCommerce product data if it's a product URL
7223		- if ($is_product_url && class_exists('WooCommerce')) {
7224		- $product_content = $this->mxchat_extract_woocommerce_product_content($url);
7225		-
7226		- if (!empty($product_content)) {
7227		- // Successfully extracted WooCommerce product data with pricing
7228		- $result = MxChat_Utils::submit_content_to_db(
7229		- $product_content,
7230		- $url,
7231		- $api_key,
7232		- null,
7233		- $bot_id,
7234		- 'product'
7235		- );
7236		- return $result;
7237		- }
7238		- // If WooCommerce extraction failed, fall through to HTML extraction
7239		- }
7240		-
7241		- // Fetch URL content (fallback for non-products or when WooCommerce extraction fails)
7242		- $is_likely_pdf = (strtolower(pathinfo(parse_url($url, PHP_URL_PATH) ?: '', PATHINFO_EXTENSION)) === 'pdf');
7243		- $response = wp_remote_get($url, array(
7244		- 'timeout' => $is_likely_pdf ? 120 : 30,
7245		- 'redirection' => 5,
7246		- 'user-agent' => 'MxChat/1.0'
7247		- ));
7248		-
7249		- if (is_wp_error($response)) {
7250		- return $response;
7251		- }
7252		-
7253		- $response_code = wp_remote_retrieve_response_code($response);
7254		- if ($response_code !== 200) {
7255		- return new WP_Error('http_error', 'HTTP ' . $response_code . ' error for: ' . $url);
7256		- }
7257		-
7258		- // Check if URL is a PDF — expand into per-page queue items using the standard PDF pipeline
7259		- if ($this->mxchat_is_pdf_url($url, $response)) {
7260		- return $this->mxchat_expand_pdf_to_queue($url, $response, $bot_id, $queue_id);
7261		- }
7262		-
7263		- $html = wp_remote_retrieve_body($response);
7264		-
7265		- if (empty($html)) {
7266		- return new WP_Error('empty_response', 'Empty response body');
7267		- }
7268		-
7269		- // Extract and sanitize content
7270		- $content = $this->mxchat_extract_main_content($html);
7271		- $sanitized = $this->mxchat_sanitize_content_for_api($content);
7272		-
7273		- if (empty($sanitized)) {
7274		- // Not an error - just no content found (maybe a redirect or empty page)
7275		- return false;
7276		- }
7277		-
7278		- // Submit to database with content_type
7279		- $result = MxChat_Utils::submit_content_to_db(
7280		- $sanitized,
7281		- $url,
7282		- $api_key,
7283		- null,
7284		- $bot_id,
7285		- $content_type
7286		- );
7287		-
7288		- return $result;
7289		-}
7290		-
7291		-/**
7292		- * Expand a PDF URL into per-page queue items using the standard PDF pipeline.
7293		- * Called when a sitemap URL turns out to be a PDF — downloads, parses page count,
7294		- * and adds pdf_page items to the same queue so they process with full progress tracking.
7295		- */
7296		-private function mxchat_expand_pdf_to_queue($pdf_url, $response, $bot_id = 'default', $queue_id = '') {
7297		- set_time_limit(120); // PDFs need extra time for download + parsing
7298		-
7299		- $upload_dir = wp_upload_dir();
7300		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
7301		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
7302		-
7303		- $response_body = wp_remote_retrieve_body($response);
7304		- if (empty($response_body)) {
7305		- return new WP_Error('empty_pdf', 'Empty PDF response for: ' . $pdf_url);
7306		- }
7307		-
7308		- if (!wp_mkdir_p(dirname($pdf_path))) {
7309		- return new WP_Error('dir_error', 'Failed to create upload directory');
7310		- }
7311		-
7312		- file_put_contents($pdf_path, $response_body);
7313		-
7314		- if (!file_exists($pdf_path)) {
7315		- return new WP_Error('save_error', 'Failed to save PDF file');
7316		- }
7317		-
7318		- try {
7319		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
7320		-
7321		- if ($total_pages === false || $total_pages < 1) {
7322		- wp_delete_file($pdf_path);
7323		- return new WP_Error('no_pages', 'PDF has no pages: ' . $pdf_url);
7324		- }
7325		-
7326		- // Build per-page items identical to mxchat_handle_pdf_for_knowledge_base
7327		- $pages = array();
7328		- for ($i = 1; $i <= $total_pages; $i++) {
7329		- $pages[] = array(
7330		- 'pdf_path' => $pdf_path,
7331		- 'pdf_url' => $pdf_url,
7332		- 'page_number' => $i,
7333		- 'total_pages' => $total_pages
7334		- );
7335		- }
7336		-
7337		- // Add pdf_page items to the SAME queue so the JS picks them up automatically
7338		- if (!empty($queue_id)) {
7339		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
7340		- } else {
7341		- // Fallback: create a new PDF queue (shouldn't happen in sitemap flow)
7342		- $new_queue_id = 'pdf_' . md5($pdf_url . time());
7343		- $queued_count = $this->mxchat_add_to_queue($new_queue_id, 'pdf_page', $pages, $bot_id);
7344		- $this->mxchat_set_queue_meta($new_queue_id, 'source_url', $pdf_url);
7345		- $this->mxchat_set_queue_meta($new_queue_id, 'queue_type', 'pdf');
7346		- $this->mxchat_set_queue_meta($new_queue_id, 'total_items', $total_pages);
7347		- $this->mxchat_set_queue_meta($new_queue_id, 'bot_id', $bot_id);
7348		- $this->mxchat_set_queue_meta($new_queue_id, 'pdf_path', $pdf_path);
7349		- $this->mxchat_set_queue_meta($new_queue_id, 'created_at', current_time('mysql'));
7350		- }
7351		-
7352		- if ($queued_count === 0) {
7353		- wp_delete_file($pdf_path);
7354		- return new WP_Error('queue_error', 'Failed to add PDF pages to queue');
7355		- }
7356		-
7357		- // Return true so the original URL item is marked complete
7358		- // The new pdf_page items will be processed in subsequent batches
7359		- return true;
7360		-
7361		- } catch (Exception $e) {
7362		- if (file_exists($pdf_path)) {
7363		- wp_delete_file($pdf_path);
7364		- }
7365		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
7366		- }
7367		-}
7368		-
7369		-/**
7370		- * Legacy: Process a PDF URL inline during sitemap queue processing.
7371		- * @deprecated Use mxchat_expand_pdf_to_queue instead — kept for reference only.
7372		- */
7373		-private function mxchat_process_pdf_url_inline($pdf_url, $response, $api_key, $bot_id = 'default') {
7374		- set_time_limit(120); // PDFs need more time — downloading + parsing all pages
7375		-
7376		- $upload_dir = wp_upload_dir();
7377		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
7378		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
7379		-
7380		- $response_body = wp_remote_retrieve_body($response);
7381		- if (empty($response_body)) {
7382		- return new WP_Error('empty_pdf', 'Empty PDF response');
7383		- }
7384		-
7385		- if (!wp_mkdir_p(dirname($pdf_path))) {
7386		- return new WP_Error('dir_error', 'Failed to create upload directory');
7387		- }
7388		-
7389		- file_put_contents($pdf_path, $response_body);
7390		-
7391		- if (!file_exists($pdf_path)) {
7392		- return new WP_Error('save_error', 'Failed to save PDF file');
7393		- }
7394		-
7395		- try {
7396		- mxchat_load_pdf_parser();
7397		- $parser = new \Smalot\PdfParser\Parser();
7398		- $pdf = $parser->parseFile($pdf_path);
7399		- $pages = $pdf->getPages();
7400		- $total_pages = count($pages);
7401		-
7402		- if ($total_pages < 1) {
7403		- wp_delete_file($pdf_path);
7404		- return new WP_Error('no_pages', 'PDF has no pages');
7405		- }
7406		-
7407		- $processed = 0;
7408		- $skipped_pages = array();
7409		-
7410		- for ($i = 0; $i < $total_pages; $i++) {
7411		- $page_num = $i + 1;
7412		- $text = $pages[$i]->getText();
7413		- if (empty($text)) {
7414		- $skipped_pages[] = 'Page ' . $page_num . ': No text could be extracted — page may contain only images, links, or non-standard encoding';
7415		- continue;
7416		- }
7417		-
7418		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
7419		- if (empty($sanitized)) {
7420		- $skipped_pages[] = 'Page ' . $page_num . ': Text was extracted but contained only special characters, control codes, or unsupported content';
7421		- continue;
7422		- }
7423		-
7424		- $metadata = array(
7425		- 'document_type' => 'pdf',
7426		- 'total_pages' => $total_pages,
7427		- 'current_page' => $page_num,
7428		- 'source_url' => $pdf_url,
7429		- );
7430		-
7431		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
7432		- $page_url = esc_url($pdf_url . '#page=' . $page_num);
7433		-
7434		- MxChat_Utils::submit_content_to_db(
7435		- $content_with_metadata,
7436		- $page_url,
7437		- $api_key,
7438		- null,
7439		- $bot_id,
7440		- 'pdf'
7441		- );
7442		-
7443		- $processed++;
7444		- }
7445		-
7446		- // Clean up the temp PDF file
7447		- wp_delete_file($pdf_path);
7448		-
7449		- if (!empty($skipped_pages)) {
7450		- error_log('MxChat PDF: Skipped ' . count($skipped_pages) . ' of ' . $total_pages . ' pages: ' . implode('; ', $skipped_pages));
7451		- }
7452		-
7453		- return $processed > 0 ? true : false;
7454		-
7455		- } catch (Exception $e) {
7456		- if (file_exists($pdf_path)) {
7457		- wp_delete_file($pdf_path);
7458		- }
7459		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
7460		- }
7461		-}
7462		-
7463		-/**
7464		- * Extract WooCommerce product content including pricing
7465		- *
7466		- * @param string $url The product URL
7467		- * @return string|false Product content with pricing, or false if not found
7468		- */
7469		-private function mxchat_extract_woocommerce_product_content($url) {
7470		- // Try to get product ID from URL
7471		- $product_id = url_to_postid($url);
7472		-
7473		- // If url_to_postid fails, try to extract from URL pattern
7474		- if (!$product_id) {
7475		- $product_slug = '';
7476		-
7477		- // Handle pretty permalinks: /product/product-name/
7478		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
7479		- $product_slug = $matches[1];
7480		- }
7481		-
7482		- if (!empty($product_slug)) {
7483		- $product_post = get_page_by_path($product_slug, OBJECT, 'product');
7484		- if ($product_post) {
7485		- $product_id = $product_post->ID;
7486		- }
7487		- }
7488		- }
7489		-
7490		- if (!$product_id) {
7491		- return false;
7492		- }
7493		-
7494		- // Get WooCommerce product object
7495		- $product = wc_get_product($product_id);
7496		-
7497		- if (!$product) {
7498		- return false;
7499		- }
7500		-
7501		- // Build product content with pricing (similar to mxchat_store_product_embedding)
7502		- $title = $product->get_name();
7503		- $description = $product->get_description();
7504		- $short_description = $product->get_short_description();
7505		- $sku = $product->get_sku();
7506		-
7507		- // Get pricing information
7508		- $regular_price = $product->get_regular_price();
7509		- $sale_price = $product->get_sale_price();
7510		- $price = $product->get_price(); // Current active price
7511		-
7512		- // Get currency symbol
7513		- $currency_symbol = get_woocommerce_currency_symbol();
7514		-
7515		- // Format content
7516		- $content = $title . "\n\n";
7517		-
7518		- if (!empty($short_description)) {
7519		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
7520		- }
7521		-
7522		- if (!empty($description)) {
7523		- $content .= wp_strip_all_tags($description) . "\n\n";
7524		- }
7525		-
7526		- // Add pricing information
7527		- if (!empty($regular_price)) {
7528		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
7529		- } elseif (!empty($price)) {
7530		- $content .= "Price: " . $currency_symbol . $price . "\n";
7531		- }
7532		-
7533		- if (!empty($sale_price) && $sale_price !== $regular_price) {
7534		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
7535		- }
7536		-
7537		- // Handle variable products - show price range
7538		- if ($product->is_type('variable')) {
7539		- $min_price = $product->get_variation_price('min');
7540		- $max_price = $product->get_variation_price('max');
7541		- if ($min_price !== $max_price) {
7542		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
7543		- }
7544		- }
7545		-
7546		- if (!empty($sku)) {
7547		- $content .= "SKU: " . $sku . "\n";
7548		- }
7549		-
7550		- // Get product categories
7551		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
7552		- if (!empty($categories) && !is_wp_error($categories)) {
7553		- $content .= "Categories: " . implode(', ', $categories) . "\n";
7554		- }
7555		-
7556		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
7557		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
7558		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
7559		- foreach ($custom_tabs as $tab) {
7560		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
7561		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
7562		-
7563		- if (!empty($tab_title) && !empty($tab_content)) {
7564		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
7565		- }
7566		- }
7567		- }
7568		-
7569		- // Also check for reusable/saved tabs applied to this product
7570		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
7571		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
7572		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
7573		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
7574		- foreach ($applied_saved_tabs as $saved_tab_id) {
7575		- if (isset($saved_tabs[$saved_tab_id])) {
7576		- $tab = $saved_tabs[$saved_tab_id];
7577		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
7578		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
7579		-
7580		- if (!empty($tab_title) && !empty($tab_content)) {
7581		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
7582		- }
7583		- }
7584		- }
7585		- }
7586		- }
7587		-
7588		- return $this->mxchat_sanitize_content_for_api($content);
7589		-}
7590		-
7591		-/**
7592		- * Process a PDF page from the queue
7593		- */
7594		-private function mxchat_process_queue_pdf_page($item_data, $bot_id = 'default') {
7595		- $pdf_path = isset($item_data['pdf_path']) ? $item_data['pdf_path'] : '';
7596		- $pdf_url = isset($item_data['pdf_url']) ? $item_data['pdf_url'] : '';
7597		- $page_number = isset($item_data['page_number']) ? absint($item_data['page_number']) : 0;
7598		- $total_pages = isset($item_data['total_pages']) ? absint($item_data['total_pages']) : 0;
7599		-
7600		- if (empty($pdf_path) || !file_exists($pdf_path)) {
7601		- return new WP_Error('pdf_not_found', 'PDF file not found: ' . $pdf_path);
7602		- }
7603		-
7604		- if ($page_number < 1) {
7605		- return new WP_Error('invalid_page', 'Invalid page number');
7606		- }
7607		-
7608		- try {
7609		- mxchat_load_pdf_parser();
7610		- $parser = new \Smalot\PdfParser\Parser();
7611		- $pdf = $parser->parseFile($pdf_path);
7612		- $pages = $pdf->getPages();
7613		-
7614		- if (!isset($pages[$page_number - 1])) {
7615		- return new WP_Error('page_not_found', 'Page ' . $page_number . ' not found in PDF');
7616		- }
7617		-
7618		- $text = $pages[$page_number - 1]->getText();
7619		-
7620		- if (empty($text)) {
7621		- return new WP_Error('empty_page', 'Page ' . $page_number . ': No text could be extracted — page may contain only images, links, or non-standard encoding');
7622		- }
7623		-
7624		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
7625		-
7626		- if (empty($sanitized)) {
7627		- return new WP_Error('empty_after_sanitization', 'Page ' . $page_number . ': Text was extracted but contained only special characters, control codes, or unsupported content that was removed during cleanup');
7628		- }
7629		-
7630		- // Create metadata
7631		- $metadata = array(
7632		- 'document_type' => 'pdf',
7633		- 'total_pages' => $total_pages,
7634		- 'current_page' => $page_number,
7635		- 'source_url' => $pdf_url
7636		- );
7637		-
7638		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
7639		- $page_url = esc_url($pdf_url . "#page=" . $page_number);
7640		-
7641		- // Get bot-specific API key
7642		- $bot_options = $this->get_bot_options($bot_id);
7643		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
7644		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
7645		-
7646		- if (strpos($selected_model, 'voyage') === 0) {
7647		- $api_key = $options['voyage_api_key'] ?? '';
7648		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
7649		- $api_key = $options['gemini_api_key'] ?? '';
7650		- } else {
7651		- $api_key = $options['api_key'] ?? '';
7652		- }
7653		-
7654		- if (empty($api_key)) {
7655		- return new WP_Error('no_api_key', 'API key not configured for bot: ' . $bot_id);
7656		- }
7657		-
7658		- // Submit to database - UPDATED 2.5.6: Added content_type 'pdf'
7659		- $result = MxChat_Utils::submit_content_to_db(
7660		- $content_with_metadata,
7661		- $page_url,
7662		- $api_key,
7663		- null,
7664		- $bot_id,
7665		- 'pdf'
7666		- );
7667		-
7668		- return $result;
7669		-
7670		- } catch (Exception $e) {
7671		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
7672		- }
7673		-}
7674		-
7675		-/**
7676		- * AJAX: Get queue processing status
7677		- */
7678		-public function ajax_mxchat_get_queue_status() {
7679		- // Verify nonce and permissions
7680		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7681		-
7682		- if (!current_user_can('manage_options')) {
7683		- wp_send_json_error('Unauthorized access');
7684		- }
7685		-
7686		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7687		-
7688		- if (empty($queue_id)) {
7689		- wp_send_json_error('Missing queue ID');
7690		- }
7691		-
7692		- global $wpdb;
7693		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7694		-
7695		- // Get counts by status
7696		- $counts = $wpdb->get_results($wpdb->prepare(
7697		- "SELECT status, COUNT(*) as count
7698		- FROM $table_name
7699		- WHERE queue_id = %s
7700		- GROUP BY status",
7701		- $queue_id
7702		- ), OBJECT_K);
7703		-
7704		- $total = 0;
7705		- $completed = 0;
7706		- $failed = 0;
7707		- $processing = 0;
7708		- $pending = 0;
7709		-
7710		- foreach ($counts as $status => $data) {
7711		- $count = absint($data->count);
7712		- $total += $count;
7713		-
7714		- switch ($status) {
7715		- case 'completed':
7716		- $completed = $count;
7717		- break;
7718		- case 'failed':
7719		- $failed = $count;
7720		- break;
7721		- case 'processing':
7722		- $processing = $count;
7723		- break;
7724		- case 'pending':
7725		- $pending = $count;
7726		- break;
7727		- }
7728		- }
7729		-
7730		- // Calculate percentage
7731		- $percentage = $total > 0 ? round((($completed + $failed) / $total) * 100) : 0;
7732		-
7733		- // Get failed items details (include all failed items, not just those that exhausted retries)
7734		- $failed_items = array();
7735		- if ($failed > 0) {
7736		- $failed_items = $wpdb->get_results($wpdb->prepare(
7737		- "SELECT item_type, item_data, error_message, attempts
7738		- FROM $table_name
7739		- WHERE queue_id = %s
7740		- AND status = 'failed'
7741		- ORDER BY id DESC
7742		- LIMIT 50",
7743		- $queue_id
7744		- ));
7745		- }
7746		-
7747		- // Get queue metadata
7748		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
7749		- $queue_type = $this->mxchat_get_queue_meta($queue_id, 'queue_type');
7750		-
7751		- // Determine if queue is complete
7752		- $is_complete = ($pending === 0 && $processing === 0);
7753		-
7754		- wp_send_json_success(array(
7755		- 'queue_id' => $queue_id,
7756		- 'queue_type' => $queue_type,
7757		- 'source_url' => $source_url,
7758		- 'total' => $total,
7759		- 'completed' => $completed,
7760		- 'failed' => $failed,
7761		- 'processing' => $processing,
7762		- 'pending' => $pending,
7763		- 'percentage' => $percentage,
7764		- 'is_complete' => $is_complete,
7765		- 'failed_items' => $failed_items,
7766		- 'status' => $is_complete ? 'complete' : 'processing'
7767		- ));
7768		-}
7769		-
7770		-/**
7771		- * AJAX: Clear completed queue
7772		- */
7773		-public function ajax_mxchat_clear_queue() {
7774		- // Verify nonce and permissions
7775		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7776		-
7777		- if (!current_user_can('manage_options')) {
7778		- wp_send_json_error('Unauthorized access');
7779		- }
7780		-
7781		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7782		-
7783		- if (empty($queue_id)) {
7784		- wp_send_json_error('Missing queue ID');
7785		- }
7786		-
7787		- global $wpdb;
7788		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7789		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7790		-
7791		- // Delete queue items
7792		- $wpdb->delete(
7793		- $table_name,
7794		- array('queue_id' => $queue_id),
7795		- array('%s')
7796		- );
7797		-
7798		- // Delete queue metadata
7799		- $wpdb->delete(
7800		- $meta_table,
7801		- array('queue_id' => $queue_id),
7802		- array('%s')
7803		- );
7804		-
7805		- wp_send_json_success(array(
7806		- 'message' => 'Queue cleared successfully'
7807		- ));
7808		-}
7809		-
7810		-/**
7811		- * AJAX: Retry failed items in queue
7812		- */
7813		-public function ajax_mxchat_retry_failed() {
7814		- // Verify nonce and permissions
7815		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7816		-
7817		- if (!current_user_can('manage_options')) {
7818		- wp_send_json_error('Unauthorized access');
7819		- }
7820		-
7821		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7822		-
7823		- if (empty($queue_id)) {
7824		- wp_send_json_error('Missing queue ID');
7825		- }
7826		-
7827		- global $wpdb;
7828		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7829		-
7830		- // Reset failed items to pending and reset attempt count
7831		- $updated = $wpdb->update(
7832		- $table_name,
7833		- array(
7834		- 'status' => 'pending',
7835		- 'attempts' => 0,
7836		- 'error_message' => null
7837		- ),
7838		- array(
7839		- 'queue_id' => $queue_id,
7840		- 'status' => 'failed'
7841		- ),
7842		- array('%s', '%d', '%s'),
7843		- array('%s', '%s')
7844		- );
7845		-
7846		- wp_send_json_success(array(
7847		- 'message' => 'Reset ' . $updated . ' failed items for retry',
7848		- 'reset_count' => $updated
7849		- ));
7850		-}
7851		-
7852		-
7853		-public function ajax_mxchat_mark_queue_complete() {
7854		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7855		-
7856		- if (!current_user_can('manage_options')) {
7857		- wp_send_json_error('Unauthorized access');
7858		- }
7859		-
7860		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7861		-
7862		- if (empty($queue_id)) {
7863		- wp_send_json_error('Missing queue ID');
7864		- }
7865		-
7866		- // Clear active queue transients
7867		- if (strpos($queue_id, 'sitemap_') === 0) {
7868		- delete_transient('mxchat_active_queue_sitemap');
7869		- } else if (strpos($queue_id, 'pdf_') === 0) {
7870		- delete_transient('mxchat_active_queue_pdf');
7871		- }
7872		-
7873		- wp_send_json_success(array('message' => 'Queue marked as complete'));
7874		-}
7875		-
7876	3406
7877	3407	// ========================================
7878	3408	// STATIC ACCESS METHODS
7879	3409	// ========================================