← All changes | admin/class-knowledge-manager.php +1612 -6141 3.2.4 → 2.2.3 View file →

		@@ -15,72 +15,49 @@
15	15
16	16	/**
17	17	* Constructor - Register hooks for content processing
18	18	*/
19		-public function __construct() {
20		- $this->options = get_option('mxchat_options', array());
21		- $this->mxchat_init_hooks();
	19	+ public function __construct() {
	20	+ $this->options = get_option('mxchat_options', array());
	21	+ $this->mxchat_init_hooks();
	22	+ }
22	23
23		- $this->mxchat_init_role_hooks();
24		-}
25		-
26		-/**
27		- * Initialize WordPress hooks for content processing
28		- *
29		- */
30		-private function mxchat_init_hooks() {
31		- // Admin post handlers for form submissions
32		- add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
33		- add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
34		- add_action('admin_post_mxchat_submit_pdf_file', array($this, 'mxchat_handle_pdf_file_submission'));
35		- add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
36		-
37		- // AJAX handlers for real-time processing and status updates
38		- add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
39		- add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status'));
40		- add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
41		- add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
42		- add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
43		- add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
44		- add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
45		- add_action('wp_ajax_mxchat_delete_chunks_by_url', array($this, 'ajax_mxchat_delete_chunks_by_url'));
46		- add_action('wp_ajax_mxchat_delete_wordpress_prompt', array($this, 'ajax_mxchat_delete_wordpress_prompt'));
47		- add_action('wp_ajax_mxchat_bulk_delete_knowledge', array($this, 'ajax_mxchat_bulk_delete_knowledge'));
48		- add_action('wp_ajax_mxchat_update_role_restriction', array($this, 'ajax_mxchat_update_role_restriction'));
49		-
50		- // Queue-based processing AJAX handlers
51		- add_action('wp_ajax_mxchat_get_next_queue_item', array($this, 'ajax_mxchat_get_next_queue_item'));
52		- add_action('wp_ajax_mxchat_process_queue_item', array($this, 'ajax_mxchat_process_queue_item'));
53		- add_action('wp_ajax_mxchat_get_queue_status', array($this, 'ajax_mxchat_get_queue_status'));
54		- add_action('wp_ajax_mxchat_clear_queue', array($this, 'ajax_mxchat_clear_queue'));
55		- add_action('wp_ajax_mxchat_retry_failed', array($this, 'ajax_mxchat_retry_failed'));
56		- add_action('wp_ajax_mxchat_get_recent_entries', array($this, 'ajax_mxchat_get_recent_entries'));
57		- add_action('wp_ajax_mxchat_detect_sitemaps', array($this, 'ajax_mxchat_detect_sitemaps'));
58		- add_action('wp_ajax_mxchat_refresh_pinecone_entries', array($this, 'ajax_mxchat_refresh_pinecone_entries'));
59		- add_action('wp_ajax_mxchat_paginate_entries', array($this, 'ajax_mxchat_paginate_entries'));
60		- add_action('wp_ajax_mxchat_get_entry_content', array($this, 'ajax_mxchat_get_entry_content'));
61		- add_action('wp_ajax_mxchat_save_entry_content', array($this, 'ajax_mxchat_save_entry_content'));
	24	+ /**
	25	+ * Initialize WordPress hooks for content processing
	26	+ */
	27	+ private function mxchat_init_hooks() {
	28	+ // Admin post handlers for form submissions
	29	+ add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
	30	+ add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
	31	+ add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
	32	+
	33	+ // AJAX handlers for real-time processing and status updates
	34	+ add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
	35	+ add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status')); // NEW
	36	+ add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
	37	+ add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
	38	+ add_action('wp_ajax_mxchat_manual_batch_process', array($this, 'ajax_manual_batch_process'));
	39	+ add_action('mxchat_delete_content', array($this, 'mxchat_delete_from_pinecone_by_url'), 10, 1);
62	40
63		- // WordPress post management hooks
64		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
65		- add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
66		- add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
67		- add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
	41	+
	42	+ // Cron handlers for background processing
	43	+ add_action('mxchat_process_sitemap_urls', array($this, 'mxchat_process_sitemap_urls_cron'), 10, 5);
	44	+ add_action('mxchat_process_pdf_pages', array($this, 'mxchat_process_pdf_pages_cron'), 10, 5);
	45	+
	46	+ // WordPress post management hooks
	47	+ add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
	48	+ add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
	49	+ add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
	50	+ add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
68	51
69		- // ACF hook - fires AFTER ACF fields are saved, ensuring ACF data is available
70		- // Priority 20 to run after ACF's own save (which runs at priority 10)
71		- add_action('acf/save_post', array($this, 'mxchat_handle_acf_save'), 20);
72		-
73		- add_action('wp_ajax_mxchat_mark_queue_complete', array($this, 'ajax_mxchat_mark_queue_complete'));
74		-
75		- // WooCommerce product hooks (if WooCommerce is active)
76		- if (class_exists('WooCommerce')) {
77		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
78		- add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
79		- add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
80		- add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
	52	+ // WooCommerce product hooks (if WooCommerce is active)
	53	+ if (class_exists('WooCommerce')) {
	54	+ add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
	55	+ add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
	56	+ add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
	57	+ }
	58	+
81	59	}
82		-}
83	60
84	61	/**
85	62	* Get current options (refreshed)
86	63	*/
		@@ -90,9 +67,315 @@
90	67	}
91	68	return $this->options;
92	69	}
93	70
	71	+
	72	+ /**
	73	+ * Handle manual batch processing via AJAX
	74	+ */
	75	+public function ajax_manual_batch_process() {
	76	+ try {
	77	+ // Verify nonce and permissions
	78	+ check_ajax_referer('mxchat_status_nonce', 'nonce');
	79	+
	80	+ if (!current_user_can('manage_options')) {
	81	+ wp_send_json_error('Unauthorized access');
	82	+ }
	83	+
	84	+ $process_type = sanitize_text_field($_POST['process_type'] ?? '');
	85	+ $url = sanitize_text_field($_POST['url'] ?? '');
	86	+
	87	+ if (empty($process_type) || empty($url)) {
	88	+ wp_send_json_error('Missing required parameters');
	89	+ }
	90	+
	91	+ $processed = 0;
	92	+
	93	+ if ($process_type === 'pdf') {
	94	+ $processed = $this->mxchat_manual_process_pdf_batch($url);
	95	+ } elseif ($process_type === 'sitemap') {
	96	+ $processed = $this->mxchat_manual_process_sitemap_batch($url);
	97	+ }
	98	+
	99	+ if ($processed > 0) {
	100	+ wp_send_json_success(array(
	101	+ 'message' => "Processed {$processed} items successfully",
	102	+ 'processed' => $processed
	103	+ ));
	104	+ } else {
	105	+ wp_send_json_error('No items were processed');
	106	+ }
	107	+
	108	+ } catch (Exception $e) {
	109	+ //error_log('Manual batch process error: ' . $e->getMessage());
	110	+ wp_send_json_error('Processing failed: ' . $e->getMessage());
	111	+ }
	112	+}
94	113
	114	+/**
	115	+ * Process a small PDF batch manually - DIRECT PROCESSING
	116	+ */
	117	+private function mxchat_manual_process_pdf_batch($pdf_url) {
	118	+ try {
	119	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	120	+ $status = get_transient($status_key);
	121	+
	122	+ if (!$status || $status['status'] !== 'processing') {
	123	+ //error_log('Manual PDF: No processing status found');
	124	+ return 0;
	125	+ }
	126	+
	127	+ //error_log('Manual PDF: Starting direct processing for ' . $pdf_url);
	128	+
	129	+ // Get current progress
	130	+ $current_page = $status['processed_pages'] ?? 0;
	131	+ $total_pages = $status['total_pages'] ?? 0;
	132	+
	133	+ if ($current_page >= $total_pages) {
	134	+ //error_log('Manual PDF: Already completed');
	135	+ return 0;
	136	+ }
	137	+
	138	+ // Try to download the PDF again for processing
	139	+ $response = wp_remote_get($pdf_url, array('timeout' => 30));
	140	+
	141	+ if (is_wp_error($response)) {
	142	+ //error_log('Manual PDF: Failed to download PDF: ' . $response->get_error_message());
	143	+ return 0;
	144	+ }
	145	+
	146	+ $pdf_content = wp_remote_retrieve_body($response);
	147	+ if (empty($pdf_content)) {
	148	+ //error_log('Manual PDF: Empty PDF content');
	149	+ return 0;
	150	+ }
	151	+
	152	+ // Save PDF temporarily
	153	+ $upload_dir = wp_upload_dir();
	154	+ $temp_pdf_path = trailingslashit($upload_dir['path']) . 'temp_manual_' . time() . '.pdf';
	155	+ file_put_contents($temp_pdf_path, $pdf_content);
	156	+
	157	+ // Process 2 pages directly
	158	+ $processed = $this->mxchat_process_pdf_pages_direct($temp_pdf_path, $pdf_url, $current_page, 5);
	159	+
	160	+ // Clean up temp file
	161	+ if (file_exists($temp_pdf_path)) {
	162	+ wp_delete_file($temp_pdf_path);
	163	+ }
	164	+
	165	+ //error_log('Manual PDF: Processed ' . $processed . ' pages');
	166	+ return $processed;
	167	+
	168	+ } catch (Exception $e) {
	169	+ //error_log('Manual PDF batch error: ' . $e->getMessage());
	170	+ return 0;
	171	+ }
	172	+}
	173	+
	174	+/**
	175	+ * Process PDF pages directly without cron
	176	+ */
	177	+private function mxchat_process_pdf_pages_direct($pdf_path, $pdf_url, $start_page, $batch_size) {
	178	+ try {
	179	+ if (!file_exists($pdf_path)) {
	180	+ //error_log('Direct PDF: File not found at ' . $pdf_path);
	181	+ return 0;
	182	+ }
	183	+
	184	+ $parser = new \Smalot\PdfParser\Parser();
	185	+ $pdf = $parser->parseFile($pdf_path);
	186	+ $pages = $pdf->getPages();
	187	+
	188	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	189	+ $status = get_transient($status_key);
	190	+
	191	+ if (!$status) {
	192	+ return 0;
	193	+ }
	194	+
	195	+ $options = get_option('mxchat_options');
	196	+ $api_key = $options['api_key'] ?? '';
	197	+
	198	+ if (empty($api_key)) {
	199	+ //error_log('Direct PDF: No API key');
	200	+ return 0;
	201	+ }
	202	+
	203	+ $processed = 0;
	204	+ $end_page = min($start_page + $batch_size, count($pages));
	205	+
	206	+ for ($i = $start_page; $i < $end_page; $i++) {
	207	+ try {
	208	+ $page_number = $i + 1;
	209	+ $text = $pages[$i]->getText();
	210	+
	211	+ if (empty($text)) {
	212	+ //error_log('Direct PDF: Empty text on page ' . $page_number);
	213	+ continue;
	214	+ }
	215	+
	216	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
	217	+ if (empty($sanitized_content)) {
	218	+ //error_log('Direct PDF: No content after sanitization on page ' . $page_number);
	219	+ continue;
	220	+ }
	221	+
	222	+ // Generate embedding
	223	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	224	+ if (is_string($embedding_vector)) {
	225	+ //error_log('Direct PDF: Embedding failed on page ' . $page_number . ': ' . $embedding_vector);
	226	+ continue;
	227	+ }
	228	+
	229	+ // Create metadata
	230	+ $metadata = array(
	231	+ 'document_type' => 'pdf',
	232	+ 'total_pages' => count($pages),
	233	+ 'current_page' => $page_number,
	234	+ 'source_url' => $pdf_url
	235	+ );
	236	+
	237	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	238	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
	239	+
	240	+ // Store in database
	241	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $api_key);
	242	+
	243	+ if (is_wp_error($db_result)) {
	244	+ //error_log('Direct PDF: DB error on page ' . $page_number . ': ' . $db_result->get_error_message());
	245	+ continue;
	246	+ }
	247	+
	248	+ $processed++;
	249	+ //error_log('Direct PDF: Successfully processed page ' . $page_number);
	250	+
	251	+ // Update status
	252	+ $status['processed_pages'] = $i + 1;
	253	+ $status['last_update'] = time();
	254	+ $status['percentage'] = round(($status['processed_pages'] / $status['total_pages']) * 100);
	255	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	256	+
	257	+ } catch (Exception $e) {
	258	+ //error_log('Direct PDF: Error processing page ' . ($i + 1) . ': ' . $e->getMessage());
	259	+ continue;
	260	+ }
	261	+ }
	262	+
	263	+ // Check if completed
	264	+ if ($status['processed_pages'] >= $status['total_pages']) {
	265	+ $status['status'] = 'complete';
	266	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	267	+ //error_log('Direct PDF: Processing completed');
	268	+ }
	269	+
	270	+ return $processed;
	271	+
	272	+ } catch (Exception $e) {
	273	+ //error_log('Direct PDF processing error: ' . $e->getMessage());
	274	+ return 0;
	275	+ }
	276	+}
	277	+
	278	+/**
	279	+ * Process a small sitemap batch manually - DIRECT PROCESSING
	280	+ */
	281	+private function mxchat_manual_process_sitemap_batch($sitemap_url) {
	282	+ try {
	283	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	284	+ $status = get_transient($status_key);
	285	+
	286	+ if (!$status || $status['status'] !== 'processing') {
	287	+ return 0;
	288	+ }
	289	+
	290	+ //error_log('Manual Sitemap: Starting direct processing for ' . $sitemap_url);
	291	+
	292	+ // Re-fetch the sitemap to get URLs
	293	+ $response = wp_remote_get($sitemap_url, array('timeout' => 30));
	294	+ if (is_wp_error($response)) {
	295	+ //error_log('Manual Sitemap: Failed to fetch sitemap');
	296	+ return 0;
	297	+ }
	298	+
	299	+ $sitemap_content = wp_remote_retrieve_body($response);
	300	+ $xml = simplexml_load_string($sitemap_content);
	301	+
	302	+ if (!$xml) {
	303	+ //error_log('Manual Sitemap: Invalid XML');
	304	+ return 0;
	305	+ }
	306	+
	307	+ $urls = array();
	308	+ foreach ($xml->url as $url_element) {
	309	+ $urls[] = (string)$url_element->loc;
	310	+ }
	311	+
	312	+ $current_processed = $status['processed_urls'] ?? 0;
	313	+ $batch_size = 5;
	314	+ $processed = 0;
	315	+
	316	+ // Process next 2 URLs
	317	+ for ($i = $current_processed; $i < min($current_processed + $batch_size, count($urls)); $i++) {
	318	+ $url = $urls[$i];
	319	+
	320	+ if ($this->mxchat_process_single_url_direct($url)) {
	321	+ $processed++;
	322	+ }
	323	+
	324	+ // Update status
	325	+ $status['processed_urls'] = $i + 1;
	326	+ $status['last_update'] = time();
	327	+ $status['percentage'] = round(($status['processed_urls'] / $status['total_urls']) * 100);
	328	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	329	+ }
	330	+
	331	+ // Check if completed
	332	+ if ($status['processed_urls'] >= $status['total_urls']) {
	333	+ $status['status'] = 'complete';
	334	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	335	+ }
	336	+
	337	+ //error_log('Manual Sitemap: Processed ' . $processed . ' URLs');
	338	+ return $processed;
	339	+
	340	+ } catch (Exception $e) {
	341	+ //error_log('Manual sitemap batch error: ' . $e->getMessage());
	342	+ return 0;
	343	+ }
	344	+}
	345	+
	346	+/**
	347	+ * Process a single URL directly
	348	+ */
	349	+private function mxchat_process_single_url_direct($url) {
	350	+ try {
	351	+ $response = wp_remote_get($url, array('timeout' => 30));
	352	+ if (is_wp_error($response)) {
	353	+ return false;
	354	+ }
	355	+
	356	+ $html = wp_remote_retrieve_body($response);
	357	+ $content = $this->mxchat_extract_main_content($html);
	358	+ $sanitized = $this->mxchat_sanitize_content_for_api($content);
	359	+
	360	+ if (empty($sanitized)) {
	361	+ return false;
	362	+ }
	363	+
	364	+ $options = get_option('mxchat_options');
	365	+ $api_key = $options['api_key'] ?? '';
	366	+
	367	+ $result = MxChat_Utils::submit_content_to_db($sanitized, $url, $api_key);
	368	+
	369	+ return !is_wp_error($result);
	370	+
	371	+ } catch (Exception $e) {
	372	+ //error_log('Single URL processing error: ' . $e->getMessage());
	373	+ return false;
	374	+ }
	375	+}
	376	+
	377	+
95	378	// ========================================
96	379	// MAIN CONTENT SUBMISSION HANDLERS
97	380	// ========================================
98	381
		@@ -108,18 +391,13 @@
108	391	wp_die(esc_html__('Nonce verification failed.', 'mxchat'));
109	392	}
110	393
111	394	// Sanitize the inputs.
112		- // Use wp_kses_post to allow safe HTML and wp_unslash to remove WordPress added slashes
113		- $article_content = wp_kses_post(wp_unslash($_POST['article_content']));
	395	+ $article_content = sanitize_textarea_field($_POST['article_content']);
114	396	$article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
115	397
116		- // Get bot_id from form submission
117		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
118		-
119		- // Get bot-specific options and API key
120		- $bot_options = $this->get_bot_options($bot_id);
121		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	398	+ // Get API key for submission
	399	+ $options = get_option('mxchat_options');
122	400	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
123	401
124	402	if (strpos($selected_model, 'voyage') === 0) {
125	403	$api_key = $options['voyage_api_key'] ?? '';
		@@ -137,10 +415,10 @@
137	415	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
138	416	exit;
139	417	}
140	418
141		- // Use centralized utility function with bot_id
142		- $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key, null, $bot_id);
	419	+ // Use centralized utility function for storage
	420	+ $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key);
143	421
144	422	if (is_wp_error($result)) {
145	423	set_transient('mxchat_admin_notice_error',
146	424	esc_html__('Error storing content: ', 'mxchat') . $result->get_error_message(),
		@@ -155,23 +433,17 @@
155	433
156	434	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
157	435	exit;
158	436	}
159		-
160	437	public function mxchat_is_pdf_url($url, $response) {
161	438	$content_type = wp_remote_retrieve_header($response, 'content-type');
162	439	$file_extension = strtolower(pathinfo($url, PATHINFO_EXTENSION));
163	440
164		- // Check Content-Disposition header for .pdf filename (Google Drive sends this)
165		- $disposition = wp_remote_retrieve_header($response, 'content-disposition');
166		- $has_pdf_disposition = ! empty($disposition) && stripos($disposition, '.pdf') !== false;
167		-
168		- return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf' || $has_pdf_disposition;
	441	+ return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf';
169	442	}
170		-
171		-
172		-public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response, $bot_id = 'default') {
	443	+public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response) {
173	444	if (!current_user_can('manage_options')) {
	445	+ //error_log(esc_html__('Unauthorized PDF processing attempt', 'mxchat'));
174	446	return false;
175	447	}
176	448
177	449	$pdf_url = esc_url_raw($pdf_url);
		@@ -177,8 +449,9 @@
177	449	$pdf_url = esc_url_raw($pdf_url);
178	450	$upload_dir = wp_upload_dir();
179	451
180	452	if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
	453	+ //error_log(sprintf(esc_html__('Upload directory error: %s', 'mxchat'), esc_html($upload_dir['error'])));
181	454	return false;
182	455	}
183	456
184	457	$pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
		@@ -185,12 +458,14 @@
185	458	$pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
186	459
187	460	$response_body = wp_remote_retrieve_body($response);
188	461	if (empty($response_body)) {
	462	+ //error_log(esc_html__('Empty PDF response body', 'mxchat'));
189	463	return false;
190	464	}
191	465
192	466	if (!wp_mkdir_p(dirname($pdf_path))) {
	467	+ //error_log(sprintf(esc_html__('Failed to create directory for PDF: %s', 'mxchat'), esc_html($pdf_path)));
193	468	return false;
194	469	}
195	470
196	471	try {
		@@ -199,693 +474,295 @@
199	474	if (!file_exists($pdf_path)) {
200	475	throw new Exception(__('Failed to save PDF file', 'mxchat'));
201	476	}
202	477
203		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
204		-
205		- if ($total_pages === false || $total_pages < 1) {
206		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
207		- }
	478	+ $parser = new \Smalot\PdfParser\Parser();
	479	+ $pdf = $parser->parseFile($pdf_path);
	480	+ $total_pages = absint(count($pdf->getPages()));
208	481
209		- // Create unique queue ID
210		- $queue_id = 'pdf_' . md5($pdf_url . time());
211		-
212		- // Create array of pages to process
213		- $pages = array();
214		- for ($i = 1; $i <= $total_pages; $i++) {
215		- $pages[] = array(
216		- 'pdf_path' => $pdf_path,
217		- 'pdf_url' => $pdf_url,
218		- 'page_number' => $i,
219		- 'total_pages' => $total_pages
220		- );
	482	+ if ($total_pages < 1) {
	483	+ throw new Exception(__('Invalid PDF: no pages found', 'mxchat'));
221	484	}
222	485
223		- // Add pages to queue
224		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
	486	+ wp_schedule_single_event(time(), 'mxchat_process_pdf_pages', array(
	487	+ 'pdf_path' => $pdf_path,
	488	+ 'pdf_url' => $pdf_url,
	489	+ 'total_pages' => $total_pages,
	490	+ 'batch_size' => absint(15),
	491	+ 'batch_pause' => absint(10)
	492	+ ));
225	493
226		- if ($queued_count === 0) {
227		- wp_delete_file($pdf_path);
228		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
229		- }
	494	+ $status_data = array(
	495	+ 'total_pages' => $total_pages,
	496	+ 'processed_pages' => 0,
	497	+ 'status' => 'processing',
	498	+ 'last_update' => time()
	499	+ );
230	500
231		- // Store queue metadata
232		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $pdf_url);
233		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
234		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
235		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
236		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
237		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	501	+ set_transient(
	502	+ sanitize_key('mxchat_pdf_status_' . md5($pdf_url)),
	503	+ array_map('sanitize_text_field', $status_data),
	504	+ DAY_IN_SECONDS
	505	+ );
238	506
239		- // Store queue ID in transient for status tracking
240		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
241		- set_transient('mxchat_last_pdf_url', $pdf_url, DAY_IN_SECONDS);
	507	+ return __('scheduled', 'mxchat');
242	508
243		- return 'queued';
244		-
245	509	} catch (Exception $e) {
	510	+ //error_log(sprintf(esc_html__('Error preparing PDF for processing: %s', 'mxchat'), esc_html($e->getMessage())));
246	511	if (file_exists($pdf_path)) {
247	512	wp_delete_file($pdf_path);
248	513	}
249		- return $e->getMessage();
	514	+ return false;
250	515	}
251	516	}
252	517
253		-/**
254		- * Handle direct PDF file upload from the knowledge base page
255		- */
256		-public function mxchat_handle_pdf_file_submission() {
257		- if (!isset($_POST['submit_pdf_file']) || !current_user_can('manage_options')) {
258		- wp_die(esc_html__('Unauthorized access', 'mxchat'));
259		- }
	518	+public function mxchat_process_pdf_pages_cron($pdf_path, $pdf_url, $total_pages, $batch_size, $batch_pause) {
	519	+ // Validate inputs
	520	+ $pdf_path = sanitize_text_field($pdf_path);
	521	+ $pdf_url = esc_url_raw($pdf_url);
	522	+ $total_pages = absint($total_pages);
	523	+ $batch_size = absint($batch_size);
	524	+ $batch_pause = absint($batch_pause);
260	525
261		- check_admin_referer('mxchat_submit_pdf_file_action', 'mxchat_submit_pdf_file_nonce');
262		-
263		- $redirect_url = admin_url('admin.php?page=mxchat-prompts');
264		-
265		- // Validate file upload
266		- if (empty($_FILES['pdf_file']) || $_FILES['pdf_file']['error'] !== UPLOAD_ERR_OK) {
267		- $error_code = isset($_FILES['pdf_file']['error']) ? $_FILES['pdf_file']['error'] : UPLOAD_ERR_NO_FILE;
268		- $error_messages = array(
269		- UPLOAD_ERR_INI_SIZE => __('The uploaded file exceeds the server upload_max_filesize limit.', 'mxchat'),
270		- UPLOAD_ERR_FORM_SIZE => __('The uploaded file exceeds the form MAX_FILE_SIZE limit.', 'mxchat'),
271		- UPLOAD_ERR_PARTIAL => __('The file was only partially uploaded.', 'mxchat'),
272		- UPLOAD_ERR_NO_FILE => __('No file was uploaded. Please select a PDF file.', 'mxchat'),
273		- UPLOAD_ERR_NO_TMP_DIR => __('Server missing temporary folder.', 'mxchat'),
274		- UPLOAD_ERR_CANT_WRITE => __('Server failed to write file to disk.', 'mxchat'),
275		- );
276		- $error_msg = isset($error_messages[$error_code]) ? $error_messages[$error_code] : __('Unknown upload error.', 'mxchat');
277		- set_transient('mxchat_admin_notice_error', $error_msg, 30);
278		- wp_safe_redirect(esc_url($redirect_url));
279		- exit;
280		- }
281		-
282		- $file = $_FILES['pdf_file'];
283		-
284		- // Validate MIME type
285		- $finfo = finfo_open(FILEINFO_MIME_TYPE);
286		- $mime_type = finfo_file($finfo, $file['tmp_name']);
287		- finfo_close($finfo);
288		-
289		- if ($mime_type !== 'application/pdf') {
290		- set_transient('mxchat_admin_notice_error',
291		- esc_html__('Invalid file type. Only PDF files are accepted.', 'mxchat'),
292		- 30
293		- );
294		- wp_safe_redirect(esc_url($redirect_url));
295		- exit;
296		- }
297		-
298		- // Validate extension
299		- $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
300		- if ($ext !== 'pdf') {
301		- set_transient('mxchat_admin_notice_error',
302		- esc_html__('Invalid file extension. Only .pdf files are accepted.', 'mxchat'),
303		- 30
304		- );
305		- wp_safe_redirect(esc_url($redirect_url));
306		- exit;
307		- }
308		-
309		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
310		- $original_filename = sanitize_file_name($file['name']);
311		-
312		- $upload_dir = wp_upload_dir();
313		- if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
314		- set_transient('mxchat_admin_notice_error',
315		- esc_html__('WordPress upload directory is not writable.', 'mxchat'),
316		- 30
317		- );
318		- wp_safe_redirect(esc_url($redirect_url));
319		- exit;
320		- }
321		-
322		- $pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
323		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
324		-
325		- if (!wp_mkdir_p(dirname($pdf_path))) {
326		- set_transient('mxchat_admin_notice_error',
327		- esc_html__('Failed to create upload directory.', 'mxchat'),
328		- 30
329		- );
330		- wp_safe_redirect(esc_url($redirect_url));
331		- exit;
332		- }
333		-
334		- // Move uploaded file
335		- if (!move_uploaded_file($file['tmp_name'], $pdf_path)) {
336		- set_transient('mxchat_admin_notice_error',
337		- esc_html__('Failed to save uploaded PDF file.', 'mxchat'),
338		- 30
339		- );
340		- wp_safe_redirect(esc_url($redirect_url));
341		- exit;
342		- }
343		-
344	526	try {
345		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
346		-
347		- if ($total_pages === false || $total_pages < 1) {
348		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
	527	+ if (!file_exists($pdf_path)) {
	528	+ throw new Exception(sprintf('PDF file not found at path: %s', esc_html($pdf_path)));
349	529	}
350	530
351		- // Use original filename as the source identifier
352		- $source_label = 'upload://' . $original_filename;
353		-
354		- $queue_id = 'pdf_' . md5($source_label . time());
355		-
356		- $pages = array();
357		- for ($i = 1; $i <= $total_pages; $i++) {
358		- $pages[] = array(
359		- 'pdf_path' => $pdf_path,
360		- 'pdf_url' => $source_label,
361		- 'page_number' => $i,
362		- 'total_pages' => $total_pages,
363		- );
364		- }
365		-
366		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
367		-
368		- if ($queued_count === 0) {
369		- wp_delete_file($pdf_path);
370		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
371		- }
372		-
373		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $source_label);
374		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
375		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
376		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
377		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
378		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
379		-
380		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
381		- set_transient('mxchat_last_pdf_url', $source_label, DAY_IN_SECONDS);
382		-
383		- set_transient('mxchat_admin_notice_success',
384		- sprintf(
385		- esc_html__('PDF "%s" (%d pages) queued for processing. Processing will start automatically.', 'mxchat'),
386		- esc_html($original_filename),
387		- $total_pages
388		- ),
389		- 30
390		- );
391		-
392		- } catch (Exception $e) {
393		- if (file_exists($pdf_path)) {
394		- wp_delete_file($pdf_path);
395		- }
396		- set_transient('mxchat_admin_notice_error',
397		- esc_html__('Failed to process uploaded PDF: ', 'mxchat') . esc_html($e->getMessage()),
398		- 30
399		- );
400		- }
401		-
402		- wp_safe_redirect(esc_url($redirect_url));
403		- exit;
404		-}
405		-
406		-/**
407		- * Validate PDF and count pages with multiple parser attempts
408		- */
409		-private function mxchat_validate_and_count_pdf_pages($pdf_path) {
410		- // Method 1: Try with Smalot PDF Parser (your current method)
411		- try {
412		- mxchat_load_pdf_parser();
413	531	$parser = new \Smalot\PdfParser\Parser();
414	532	$pdf = $parser->parseFile($pdf_path);
415	533	$pages = $pdf->getPages();
416		- $page_count = count($pages);
417		-
418		- if ($page_count > 0) {
419		- //error_log('PDF parsed successfully with Smalot parser: ' . $page_count . ' pages');
420		- return $page_count;
421		- }
422		- } catch (Exception $e) {
423		- //error_log('Smalot PDF parser failed: ' . $e->getMessage());
424		- }
425	534
426		- // Method 2: Try with pdfinfo command (if available)
427		- if (function_exists('shell_exec') && !$this->mxchat_is_shell_disabled()) {
428		- try {
429		- $command = 'pdfinfo ' . escapeshellarg($pdf_path) . ' 2>&1';
430		- $output = shell_exec($command);
431		-
432		- if ($output && preg_match('/Pages:\s*(\d+)/', $output, $matches)) {
433		- $page_count = intval($matches[1]);
434		- if ($page_count > 0) {
435		- //error_log('PDF parsed successfully with pdfinfo: ' . $page_count . ' pages');
436		- return $page_count;
437		- }
438		- }
439		- } catch (Exception $e) {
440		- //error_log('pdfinfo command failed: ' . $e->getMessage());
	535	+ // Get current progress
	536	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	537	+ $status = get_transient($status_key);
	538	+
	539	+ if (!$status || !is_array($status)) {
	540	+ throw new Exception('Invalid status data retrieved from transient');
441	541	}
442		- }
443	542
444		- // Method 3: Try to repair PDF and parse again
445		- try {
446		- $repaired_path = $this->mxchat_attempt_pdf_repair($pdf_path);
447		- if ($repaired_path && $repaired_path !== $pdf_path) {
448		- mxchat_load_pdf_parser();
449		- $parser = new \Smalot\PdfParser\Parser();
450		- $pdf = $parser->parseFile($repaired_path);
451		- $pages = $pdf->getPages();
452		- $page_count = count($pages);
453		-
454		- if ($page_count > 0) {
455		- // Replace original with repaired version
456		- copy($repaired_path, $pdf_path);
457		- unlink($repaired_path);
458		- //error_log('PDF repaired and parsed successfully: ' . $page_count . ' pages');
459		- return $page_count;
460		- }
461		-
462		- // Clean up repaired file if it didn't work
463		- unlink($repaired_path);
	543	+ // Initialize failed pages list if it doesn't exist
	544	+ if (!isset($status['failed_pages_list']) || !is_array($status['failed_pages_list'])) {
	545	+ $status['failed_pages_list'] = [];
464	546	}
465		- } catch (Exception $e) {
466		- //error_log('PDF repair attempt failed: ' . $e->getMessage());
467		- }
468	547
469		- // Method 4: Manual PDF structure analysis (basic page count)
470		- try {
471		- $page_count = $this->mxchat_manual_pdf_page_count($pdf_path);
472		- if ($page_count > 0) {
473		- //error_log('PDF page count determined manually: ' . $page_count . ' pages');
474		- return $page_count;
	548	+ $start_page = absint($status['processed_pages']);
	549	+ $end_page = min($start_page + $batch_size, $total_pages);
	550	+ $options = get_option('mxchat_options');
	551	+
	552	+ if (empty($options['api_key'])) {
	553	+ throw new Exception('API key is missing or invalid');
475	554	}
476		- } catch (Exception $e) {
477		- //error_log('Manual PDF analysis failed: ' . $e->getMessage());
478		- }
479	555
480		- //error_log('All PDF parsing methods failed for: ' . $pdf_path);
481		- return false;
482		-}
	556	+ $successful_pages = 0;
	557	+ $failed_pages = 0;
483	558
484		-/**
485		- * Check if shell_exec is disabled
486		- */
487		-private function mxchat_is_shell_disabled() {
488		- $disabled = explode(',', ini_get('disable_functions'));
489		- return in_array('shell_exec', $disabled);
490		-}
	559	+ for ($i = $start_page; $i < $end_page; $i++) {
	560	+ $page_number = $i + 1;
	561	+ $max_retries = 3;
	562	+ $retry_count = 0;
	563	+ $page_processed = false;
	564	+ $last_error = '';
491	565
492		-/**
493		- * Attempt to repair PDF using basic methods
494		- */
495		-private function mxchat_attempt_pdf_repair($pdf_path) {
496		- try {
497		- $content = file_get_contents($pdf_path);
498		- if (!$content) {
499		- return false;
500		- }
	566	+ while (!$page_processed && $retry_count < $max_retries) {
	567	+ try {
	568	+ $text = $pages[$i]->getText();
	569	+
	570	+ if (empty($text)) {
	571	+ throw new Exception("Empty text on page {$page_number}");
	572	+ }
	573	+
	574	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
501	575
502		- // Check if PDF starts with proper header
503		- if (substr($content, 0, 4) !== '%PDF') {
504		- // Try to find PDF header in the content
505		- $header_pos = strpos($content, '%PDF');
506		- if ($header_pos !== false && $header_pos < 1024) {
507		- // Remove junk before PDF header
508		- $content = substr($content, $header_pos);
509		- $repaired_path = $pdf_path . '.repaired';
510		- file_put_contents($repaired_path, $content);
511		- return $repaired_path;
512		- }
513		- }
	576	+ if (empty($sanitized_content)) {
	577	+ throw new Exception("No valid content after sanitization on page {$page_number}");
	578	+ }
514	579
515		- // Check for EOF marker
516		- $content = rtrim($content);
517		- if (!preg_match('/%%EOF\s*$/', $content)) {
518		- // Add EOF marker if missing
519		- $content .= "\n%%EOF";
520		- $repaired_path = $pdf_path . '.repaired';
521		- file_put_contents($repaired_path, $content);
522		- return $repaired_path;
523		- }
	580	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	581	+
	582	+ if (is_string($embedding_vector)) {
	583	+ throw new Exception("Embedding generation failed: " . $embedding_vector);
	584	+ }
	585	+
	586	+ if (!is_array($embedding_vector)) {
	587	+ throw new Exception("Embedding generation returned unexpected result type: " . gettype($embedding_vector));
	588	+ }
	589	+
	590	+ $metadata = array(
	591	+ 'document_type' => 'pdf',
	592	+ 'total_pages' => $total_pages,
	593	+ 'current_page' => $page_number,
	594	+ 'prev_page' => $i > 0 ? $i : null,
	595	+ 'next_page' => $i < ($total_pages - 1) ? $i + 2 : null,
	596	+ 'source_url' => $pdf_url
	597	+ );
524	598
525		- } catch (Exception $e) {
526		- //error_log('PDF repair error: ' . $e->getMessage());
527		- }
	599	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	600	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
528	601
529		- return false;
530		-}
	602	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $options['api_key']);
	603	+
	604	+ if (is_wp_error($db_result)) {
	605	+ throw new Exception("Database submission failed: " . $db_result->get_error_message());
	606	+ }
531	607
532		-/**
533		- * Manual PDF page counting by analyzing PDF structure
534		- */
535		-private function mxchat_manual_pdf_page_count($pdf_path) {
536		- try {
537		- $content = file_get_contents($pdf_path);
538		- if (!$content) {
539		- return 0;
540		- }
	608	+ // Success!
	609	+ $page_processed = true;
	610	+ $successful_pages++;
	611	+
	612	+ } catch (Exception $e) {
	613	+ $retry_count++;
	614	+ $last_error = $e->getMessage();
	615	+
	616	+ //error_log("PDF page {$page_number} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	617	+
	618	+ if ($retry_count < $max_retries) {
	619	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	620	+ sleep(pow(2, $retry_count - 1));
	621	+ }
	622	+ }
	623	+ }
541	624
542		- // Method 1: Count /Type /Page objects
543		- $page_count = preg_match_all('/\/Type\s*\/Page[^s]/', $content);
544		- if ($page_count > 0) {
545		- return $page_count;
546		- }
	625	+ // If page still not processed after all retries, mark as failed
	626	+ if (!$page_processed) {
	627	+ $failed_pages++;
	628	+ $status['failed_pages_list'][] = [
	629	+ 'page' => $page_number,
	630	+ 'error' => $last_error,
	631	+ 'time' => time(),
	632	+ 'retries' => $max_retries
	633	+ ];
	634	+
	635	+ // Limit failed pages list to prevent memory issues
	636	+ if (count($status['failed_pages_list']) > 50) {
	637	+ $status['failed_pages_list'] = array_slice($status['failed_pages_list'], -50);
	638	+ }
	639	+
	640	+ //error_log("PDF page {$page_number} permanently failed after {$max_retries} attempts: " . $last_error);
	641	+ }
547	642
548		- // Method 2: Look for /Count in pages object
549		- if (preg_match('/\/Type\s*\/Pages.*?\/Count\s+(\d+)/', $content, $matches)) {
550		- return intval($matches[1]);
	643	+ // Update progress
	644	+ $status['processed_pages'] = absint($page_number);
	645	+ $status['last_update'] = time();
	646	+ $status['failed_pages'] = absint($status['failed_pages'] ?? 0) + ($page_processed ? 0 : 1);
	647	+
	648	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
551	649	}
552	650
553		- // Method 3: Count page references
554		- $page_count = preg_match_all('/\d+\s+0\s+obj\s*<<[^>]*\/Type\s*\/Page/', $content);
555		- if ($page_count > 0) {
556		- return $page_count;
557		- }
558		-
559		- } catch (Exception $e) {
560		- //error_log('Manual PDF analysis error: ' . $e->getMessage());
561		- }
562		-
563		- return 0;
564		-}
565		-
566		-
567		-public function mxchat_save_inline_prompt() {
568		- // DEBUG: Log what we're receiving
569		- //error_log('=== MXCHAT DEBUG ===');
570		- //error_log('POST data: ' . print_r($_POST, true));
571		- //error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
572		-
573		- // Check for nonce security
574		- check_ajax_referer('mxchat_save_inline_nonce', '_ajax_nonce');
575		-
576		- // If we get here, nonce passed
577		- //error_log('Nonce verification PASSED');
578		-
579		- // Verify permissions
580		- if (!current_user_can('manage_options')) {
581		- wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
582		- return;
583		- }
584		-
585		- global $wpdb;
586		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
587		-
588		- // Validate and sanitize input data
589		- $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
590		- $article_content = isset($_POST['article_content']) ? wp_kses_post(wp_unslash($_POST['article_content'])) : '';
591		- $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
592		-
593		- if ($prompt_id > 0 && !empty($article_content)) {
594		- // Re-generate the embedding vector for the updated content
595		- $embedding_vector = $this->mxchat_generate_embedding($article_content);
596		- if (is_array($embedding_vector)) {
597		- // Serialize the embedding vector before storing it
598		- $embedding_vector_serialized = serialize($embedding_vector);
599		- // Update the prompt in the database
600		- $updated = $wpdb->update(
601		- $table_name,
602		- array(
603		- 'article_content' => $article_content,
604		- 'embedding_vector' => $embedding_vector_serialized,
605		- 'source_url' => $article_url,
606		- ),
607		- array('id' => $prompt_id),
608		- array('%s', '%s', '%s'),
609		- array('%d')
610		- );
611		- if ($updated !== false) {
612		- wp_send_json_success();
613		- } else {
614		- MxChat_Admin::mxchat_log_debug('knowledge_error', 'Database update failed when saving knowledge entry');
615		- wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
616		- }
	651	+ // Schedule next batch if needed
	652	+ if ($end_page < $total_pages) {
	653	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_pdf_pages', array(
	654	+ 'pdf_path' => $pdf_path,
	655	+ 'pdf_url' => $pdf_url,
	656	+ 'total_pages' => $total_pages,
	657	+ 'batch_size' => $batch_size,
	658	+ 'batch_pause' => $batch_pause
	659	+ ));
617	660	} else {
618		- MxChat_Admin::mxchat_log_debug('embedding_error', 'Embedding generation failed for knowledge entry');
619		- wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
620		- }
621		- } else {
622		- wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
623		- }
624		-}
625		-
626		-
627		-/**
628		- * AJAX: Get full content for editing — reassembles chunks if needed.
629		- * Works for both WordPress DB and Pinecone entries.
630		- */
631		-public function ajax_mxchat_get_entry_content() {
632		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
633		-
634		- if ( ! current_user_can('manage_options') ) {
635		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
636		- }
637		-
638		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
639		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
640		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
641		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
642		-
643		- if ( $data_source === 'pinecone' ) {
644		- // Pinecone: fetch vectors by source_url, reassemble chunks
645		- $content = $this->get_pinecone_entry_content( $source_url, $entry_id, $bot_id );
646		- } else {
647		- // WordPress DB
648		- $content = $this->get_wordpress_entry_content( $source_url, $entry_id );
649		- }
650		-
651		- if ( is_wp_error( $content ) ) {
652		- wp_send_json_error( array( 'message' => $content->get_error_message() ) );
653		- }
654		-
655		- wp_send_json_success( $content );
656		-}
657		-
658		-/**
659		- * Get content from WordPress DB — reassembles chunks by source_url.
660		- */
661		-private function get_wordpress_entry_content( $source_url, $entry_id ) {
662		- global $wpdb;
663		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
664		-
665		- // If we have a source_url, check for chunks
666		- if ( ! empty( $source_url ) && strpos( $source_url, 'mxchat://' ) !== 0 ) {
667		- $rows = $wpdb->get_results( $wpdb->prepare(
668		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
669		- $source_url
670		- ) );
671		-
672		- if ( $rows && count( $rows ) > 1 ) {
673		- // Multiple rows = chunked. Reassemble.
674		- $chunks = array();
675		- foreach ( $rows as $row ) {
676		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
677		- $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
678		- $chunks[ $index ] = $parsed['text'];
	661	+ // Processing complete
	662	+ $status['status'] = 'complete';
	663	+ $status['processed_pages'] = $total_pages;
	664	+
	665	+ // Add completion summary
	666	+ $status['completion_summary'] = [
	667	+ 'total_pages' => $total_pages,
	668	+ 'successful_pages' => $total_pages - absint($status['failed_pages'] ?? 0),
	669	+ 'failed_pages' => absint($status['failed_pages'] ?? 0),
	670	+ 'completion_time' => current_time('mysql')
	671	+ ];
	672	+
	673	+ // Save the completed status (don't delete it - let user dismiss manually)
	674	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	675	+
	676	+ // Clean up the temporary PDF file
	677	+ if (file_exists($pdf_path)) {
	678	+ wp_delete_file($pdf_path);
679	679	}
680		- ksort( $chunks );
681		- return array(
682		- 'content' => implode( "\n\n", $chunks ),
683		- 'source_url' => $source_url,
684		- 'is_chunked' => true,
685		- 'chunk_count' => count( $chunks ),
686		- 'content_type' => $rows[0]->content_type,
687		- );
688		- } elseif ( $rows && count( $rows ) === 1 ) {
689		- $parsed = MxChat_Chunker::parse_stored_chunk( $rows[0]->article_content );
690		- return array(
691		- 'content' => $parsed['text'],
692		- 'source_url' => $source_url,
693		- 'entry_id' => $rows[0]->id,
694		- 'is_chunked' => false,
695		- 'content_type' => $rows[0]->content_type,
696		- );
	680	+
	681	+ // DON'T delete the status transients here - let user dismiss manually
697	682	}
698		- }
699	683
700		- // Fallback: fetch by ID
701		- if ( $entry_id > 0 ) {
702		- $row = $wpdb->get_row( $wpdb->prepare(
703		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
704		- $entry_id
705		- ) );
706		- if ( $row ) {
707		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
708		- return array(
709		- 'content' => $parsed['text'],
710		- 'source_url' => $row->source_url,
711		- 'entry_id' => $row->id,
712		- 'is_chunked' => false,
713		- 'content_type' => $row->content_type,
	684	+ } catch (\Exception $e) {
	685	+ //error_log(sprintf('[MXCHAT-PDF] Error processing PDF: %s', $e->getMessage()));
	686	+
	687	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	688	+ $status = get_transient($status_key);
	689	+
	690	+ if (!$status || !is_array($status)) {
	691	+ $status = array(
	692	+ 'total_pages' => $total_pages,
	693	+ 'processed_pages' => 0,
	694	+ 'status' => 'error',
	695	+ 'error' => sanitize_text_field($e->getMessage()),
	696	+ 'last_update' => time()
714	697	);
	698	+ } else {
	699	+ $status['status'] = 'error';
	700	+ $status['error'] = sanitize_text_field($e->getMessage());
	701	+ $status['last_update'] = time();
715	702	}
716		- }
717		-
718		- return new WP_Error( 'not_found', 'Entry not found.' );
719		-}
720		-
721		-/**
722		- * Get content from Pinecone — fetches vectors by source_url, reassembles chunks.
723		- */
724		-private function get_pinecone_entry_content( $source_url, $entry_id, $bot_id ) {
725		- if ( ! class_exists('MxChat_Pinecone_Manager') ) {
726		- return new WP_Error( 'pinecone_unavailable', 'Pinecone manager not available.' );
727		- }
728		-
729		- // Get Pinecone config
730		- if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
731		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
732		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
733		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
734		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
735		- } else {
736		- $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
737		- $api_key = $bot_config['api_key'] ?? '';
738		- $host = $bot_config['host'] ?? '';
739		- $namespace = $bot_config['namespace'] ?? '';
740		- }
741		-
742		- if ( empty($host) || empty($api_key) ) {
743		- return new WP_Error( 'pinecone_config', 'Pinecone not configured.' );
744		- }
745		-
746		- // List vectors with the source_url prefix
747		- $base_id = md5( $source_url );
748		- $vector_ids = array( $base_id );
749		-
750		- // Find chunk vectors
751		- $list_url = "https://{$host}/vectors/list";
752		- $list_body = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
753		- if ( ! empty($namespace) ) {
754		- $list_body['namespace'] = $namespace;
755		- }
756		-
757		- $list_resp = wp_remote_post( $list_url, array(
758		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
759		- 'body' => wp_json_encode( $list_body ),
760		- 'timeout' => 15,
761		- ) );
762		-
763		- if ( ! is_wp_error($list_resp) ) {
764		- $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
765		- if ( ! empty($list_data['vectors']) ) {
766		- foreach ( $list_data['vectors'] as $v ) {
767		- $vector_ids[] = $v['id'];
768		- }
	703	+
	704	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	705	+
	706	+ if (file_exists($pdf_path)) {
	707	+ wp_delete_file($pdf_path);
769	708	}
770	709	}
771		-
772		- // Fetch vectors with metadata
773		- $fetch_url = "https://{$host}/vectors/fetch";
774		- $fetch_body = array( 'ids' => $vector_ids );
775		- if ( ! empty($namespace) ) {
776		- $fetch_body['namespace'] = $namespace;
777		- }
778		-
779		- $fetch_resp = wp_remote_post( $fetch_url, array(
780		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
781		- 'body' => wp_json_encode( $fetch_body ),
782		- 'timeout' => 15,
783		- ) );
784		-
785		- if ( is_wp_error($fetch_resp) ) {
786		- return new WP_Error( 'pinecone_fetch', 'Failed to fetch from Pinecone.' );
787		- }
788		-
789		- $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
790		- $vectors = $fetch_data['vectors'] ?? array();
791		-
792		- if ( empty($vectors) ) {
793		- return new WP_Error( 'not_found', 'Entry not found in Pinecone.' );
794		- }
795		-
796		- // Reassemble chunks
797		- $chunks = array();
798		- $content_type = 'content';
799		- foreach ( $vectors as $vid => $vector ) {
800		- $meta = $vector['metadata'] ?? array();
801		- $text = $meta['text'] ?? '';
802		- $index = $meta['chunk_index'] ?? 0;
803		- $content_type = $meta['type'] ?? 'content';
804		- $chunks[ intval($index) ] = $text;
805		- }
806		- ksort( $chunks );
807		-
808		- return array(
809		- 'content' => implode( "\n\n", $chunks ),
810		- 'source_url' => $source_url,
811		- 'is_chunked' => count($chunks) > 1,
812		- 'chunk_count' => count($chunks),
813		- 'content_type' => $content_type,
814		- );
815	710	}
816	711
817		-/**
818		- * AJAX: Save edited content — re-chunks and re-embeds as needed.
819		- * Works for both WordPress DB and Pinecone entries.
820		- */
821		-public function ajax_mxchat_save_entry_content() {
822		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
	712	+ public function mxchat_save_inline_prompt() {
	713	+ // Check for nonce security
	714	+ check_ajax_referer('mxchat_save_inline_nonce');
823	715
824		- if ( ! current_user_can('manage_options') ) {
825		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
826		- }
	716	+ // Verify permissions
	717	+ if (!current_user_can('manage_options')) {
	718	+ wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
	719	+ return;
	720	+ }
827	721
828		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
829		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
830		- $content = isset($_POST['content']) ? wp_kses_post( wp_unslash($_POST['content']) ) : '';
831		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
832		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
833		- $content_type = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : 'content';
	722	+ global $wpdb;
	723	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
834	724
835		- if ( empty($content) ) {
836		- wp_send_json_error( array( 'message' => 'Content cannot be empty.' ) );
837		- }
	725	+ // Validate and sanitize input data
	726	+ $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
	727	+ $article_content = isset($_POST['article_content']) ? sanitize_textarea_field($_POST['article_content']) : '';
	728	+ $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
838	729
839		- // Get the embedding API key
840		- $options = get_option('mxchat_options', array());
841		- $api_key = '';
	730	+ if ($prompt_id > 0 && !empty($article_content)) {
	731	+ // Re-generate the embedding vector for the updated content
	732	+ $embedding_vector = $this->mxchat_generate_embedding($article_content);
842	733
843		- if ( $bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager') ) {
844		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
845		- $api_key = $bot_options['api_key'] ?? '';
846		- }
847		- if ( empty($api_key) ) {
848		- $api_key = $options['api_key'] ?? '';
849		- }
	734	+ if (is_array($embedding_vector)) {
	735	+ // Serialize the embedding vector before storing it
	736	+ $embedding_vector_serialized = serialize($embedding_vector);
850	737
851		- global $wpdb;
852		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
	738	+ // Update the prompt in the database
	739	+ $updated = $wpdb->update(
	740	+ $table_name,
	741	+ array(
	742	+ 'article_content' => $article_content,
	743	+ 'embedding_vector' => $embedding_vector_serialized,
	744	+ 'source_url' => $article_url,
	745	+ ),
	746	+ array('id' => $prompt_id),
	747	+ array('%s', '%s', '%s'),
	748	+ array('%d')
	749	+ );
853	750
854		- // If source_url is empty but we have an entry_id, look it up
855		- if ( empty($source_url) && $entry_id > 0 && $data_source === 'wordpress' ) {
856		- $row = $wpdb->get_row( $wpdb->prepare( "SELECT source_url FROM {$table} WHERE id = %d", $entry_id ) );
857		- if ( $row && ! empty($row->source_url) ) {
858		- $source_url = $row->source_url;
859		- }
860		- }
	751	+ if ($updated !== false) {
	752	+ wp_send_json_success();
	753	+ } else {
	754	+ wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
	755	+ }
	756	+ } else {
	757	+ wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
	758	+ }
	759	+ } else {
	760	+ wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
	761	+ }
	762	+ }
861	763
862		- // For manual entries (no source_url or mxchat:// prefix), delete the old entry by ID first
863		- // so submit_content_to_db creates a replacement instead of a duplicate
864		- // Also treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
865		- $is_legacy_manual = !empty($source_url) && strpos($source_url, 'mxchat.ai') !== false && strpos($source_url, 'mxchat://') !== 0;
866		- if ( $entry_id > 0 && (empty($source_url) || strpos($source_url, 'mxchat://') === 0 || $is_legacy_manual) ) {
867		- $wpdb->delete( $table, array( 'id' => $entry_id ), array( '%d' ) );
868		- // Clear legacy URL so submit_content_to_db generates a unique mxchat:// identifier
869		- // instead of reusing the shared URL (which would mass-delete other entries with the same URL)
870		- if ( $is_legacy_manual ) {
871		- $source_url = '';
872		- }
873		- }
874	764
875		- // Use the existing submit_content_to_db which handles chunking, Pinecone, and WP DB
876		- $vector_id = ! empty($source_url) ? md5($source_url) : md5('mxchat_manual_' . $entry_id);
877		-
878		- // submit_content_to_db already handles: delete old chunks → re-chunk → re-embed → store
879		- $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, $vector_id, $bot_id, $content_type );
880		-
881		- if ( is_wp_error($result) ) {
882		- wp_send_json_error( array( 'message' => $result->get_error_message() ) );
883		- }
884		-
885		- wp_send_json_success( array( 'message' => 'Content saved and re-embedded successfully.' ) );
886		-}
887		-
888	765	public function mxchat_get_pdf_processing_status($pdf_url) {
889	766	$pdf_url = esc_url_raw($pdf_url);
890	767	$status = get_transient(sanitize_key('mxchat_pdf_status_' . md5($pdf_url)));
891	768
		@@ -928,18 +805,24 @@
928	805	}
929	806
930	807
931	808	public function mxchat_handle_sitemap_submission() {
	809	+ // Start logging the submission process
	810	+ //error_log('[MXCHAT-URL] ===== Starting URL submission process =====');
	811	+
932	812	// Check if the form was submitted and verify permissions
933	813	if (!isset($_POST['submit_sitemap']) || !current_user_can('manage_options')) {
	814	+ //error_log('[MXCHAT-URL] Error: Unauthorized access or form not submitted properly');
934	815	wp_die(esc_html__('Unauthorized access', 'mxchat'));
935	816	}
936	817
937	818	// Verify nonce
	819	+ //error_log('[MXCHAT-URL] Verifying nonce');
938	820	check_admin_referer('mxchat_submit_sitemap_action', 'mxchat_submit_sitemap_nonce');
939	821
940	822	// Validate URL
941	823	if (!isset($_POST['sitemap_url']) || empty($_POST['sitemap_url'])) {
	824	+ //error_log('[MXCHAT-URL] Error: Empty or missing URL');
942	825	set_transient('mxchat_admin_notice_error',
943	826	esc_html__('Please provide a valid URL.', 'mxchat'),
944	827	30
945	828	);
		@@ -947,28 +830,12 @@
947	830	exit;
948	831	}
949	832
950	833	$submitted_url = esc_url_raw($_POST['sitemap_url']);
	834	+ //error_log('[MXCHAT-URL] Processing URL: ' . $submitted_url);
951	835
952		- // Convert Google Drive sharing URLs to direct download URLs
953		- if ( strpos($submitted_url, 'drive.google.com') !== false ) {
954		- $file_id = '';
955		- if ( preg_match('/[?&]id=([a-zA-Z0-9_-]+)/', $submitted_url, $m) ) {
956		- $file_id = $m[1];
957		- } elseif ( preg_match('#/file/d/([a-zA-Z0-9_-]+)#', $submitted_url, $m) ) {
958		- $file_id = $m[1];
959		- }
960		- if ( ! empty($file_id) ) {
961		- $submitted_url = 'https://drive.google.com/uc?export=download&id=' . $file_id;
962		- }
963		- }
964		-
965		- // Get bot_id from form submission
966		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
967		-
968		- // Get bot-specific options and validate API key
969		- $bot_options = $this->get_bot_options($bot_id);
970		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	836	+ // Validate API key first
	837	+ $options = get_option('mxchat_options');
971	838	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
972	839
973	840	if (strpos($selected_model, 'voyage') === 0) {
974	841	$api_key = $options['voyage_api_key'] ?? '';
		@@ -985,26 +852,21 @@
985	852	$error_message = sprintf(
986	853	esc_html__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
987	854	$provider_name
988	855	);
	856	+ //error_log('[MXCHAT-URL] Error: ' . $error_message);
989	857	set_transient('mxchat_admin_notice_error', $error_message, 30);
	858	+ //error_log('[MXCHAT-URL] Set error transient: ' . $error_message);
990	859	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
991	860	exit;
992	861	}
993	862
994		- // Fetch URL — use browser-like headers so servers with bot protection don't block us
995		- $response = wp_remote_get($submitted_url, array(
996		- 'timeout' => 30,
997		- 'sslverify' => false,
998		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
999		- 'headers' => array(
1000		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1001		- 'Accept-Language' => 'en-US,en;q=0.9',
1002		- ),
1003		- ));
	863	+ //error_log('[MXCHAT-URL] Fetching URL content');
	864	+ $response = wp_remote_get($submitted_url, array('timeout' => 30));
1004	865
1005	866	if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
1006	867	$error_message = is_wp_error($response) ? $response->get_error_message() : 'HTTP Status: ' . wp_remote_retrieve_response_code($response);
	868	+ //error_log('[MXCHAT-URL] Error fetching URL: ' . $error_message);
1007	869	set_transient('mxchat_admin_notice_error',
1008	870	sprintf(
1009	871	esc_html__('Failed to fetch the URL: %s', 'mxchat'),
1010	872	esc_html($error_message)
		@@ -1015,11 +877,13 @@
1015	877	exit;
1016	878	}
1017	879
1018	880	$content_type = wp_remote_retrieve_header($response, 'content-type');
	881	+ //error_log('[MXCHAT-URL] Content type: ' . $content_type);
1019	882	$body_content = wp_remote_retrieve_body($response);
1020	883
1021	884	if (empty($body_content)) {
	885	+ //error_log('[MXCHAT-URL] Error: Empty response body');
1022	886	set_transient('mxchat_admin_notice_error',
1023	887	esc_html__('Empty response received from URL.', 'mxchat'),
1024	888	30
1025	889	);
		@@ -1025,21 +889,29 @@
1025	889	);
1026	890	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1027	891	exit;
1028	892	}
	893	+ //error_log('[MXCHAT-URL] Retrieved body content length: ' . strlen($body_content) . ' bytes');
1029	894
1030	895	// Handle PDF URL
1031	896	if ($this->mxchat_is_pdf_url($submitted_url, $response)) {
1032		- $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response, $bot_id);
	897	+ //error_log('[MXCHAT-URL] Detected PDF URL, handling PDF for knowledge base');
	898	+ $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response);
	899	+ //error_log('[MXCHAT-URL] PDF handling result: ' . $result);
1033	900
1034		- if ($result === 'queued') {
1035		- set_transient('mxchat_admin_notice_success',
1036		- esc_html__('PDF queued for processing. Processing will start automatically.', 'mxchat'),
	901	+ if ($result === 'scheduled') {
	902	+ set_transient(
	903	+ 'mxchat_last_pdf_url',
	904	+ sanitize_text_field($submitted_url),
	905	+ DAY_IN_SECONDS
	906	+ );
	907	+ set_transient('mxchat_admin_notice_info',
	908	+ esc_html__('PDF processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1037	909	30
1038	910	);
1039	911	} else {
1040	912	set_transient('mxchat_admin_notice_error',
1041		- esc_html__('Failed to queue PDF processing: ', 'mxchat') . esc_html($result),
	913	+ esc_html__('Failed to start PDF processing: ', 'mxchat') . esc_html($result),
1042	914	30
1043	915	);
1044	916	}
1045	917
		@@ -1048,8 +920,9 @@
1048	920	}
1049	921
1050	922	// Handle Sitemap XML
1051	923	if (strpos($content_type, 'xml') !== false || strpos($body_content, '<urlset') !== false) {
	924	+ //error_log('[MXCHAT-URL] Detected XML content, processing as sitemap');
1052	925	libxml_use_internal_errors(true);
1053	926	$xml = simplexml_load_string($body_content);
1054	927	$xml_errors = libxml_get_errors();
1055	928	libxml_clear_errors();
		@@ -1054,8 +927,15 @@
1054	927	$xml_errors = libxml_get_errors();
1055	928	libxml_clear_errors();
1056	929
1057	930	if ($xml === false || !empty($xml_errors)) {
	931	+ //error_log('[MXCHAT-URL] Error: Invalid XML format');
	932	+ if (!empty($xml_errors)) {
	933	+ foreach ($xml_errors as $error) {
	934	+ //error_log('[MXCHAT-URL] XML Error: ' . $error->message);
	935	+ }
	936	+ }
	937	+
1058	938	set_transient('mxchat_admin_notice_error',
1059	939	esc_html__('Invalid sitemap XML. Please provide a valid sitemap.', 'mxchat'),
1060	940	30
1061	941	);
		@@ -1062,18 +942,27 @@
1062	942	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1063	943	exit;
1064	944	}
1065	945
1066		- $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url, $bot_id);
	946	+ //error_log('[MXCHAT-URL] Valid XML found, handling sitemap for knowledge base');
	947	+ $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url);
	948	+ //error_log('[MXCHAT-URL] Sitemap handling result: ' . $result);
1067	949
1068		- if ($result === 'queued') {
1069		- set_transient('mxchat_admin_notice_success',
1070		- esc_html__('Sitemap queued for processing. Processing will start automatically.', 'mxchat'),
	950	+ if ($result === 'scheduled') {
	951	+ set_transient(
	952	+ 'mxchat_last_sitemap_url',
	953	+ sanitize_text_field($submitted_url),
	954	+ DAY_IN_SECONDS
	955	+ );
	956	+ set_transient('mxchat_admin_notice_info',
	957	+ esc_html__('Sitemap processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1071	958	30
1072	959	);
1073	960	} else {
	961	+ // Return to the admin page without a redirect for better error display
	962	+ // The error is already stored in the sitemap status transient
1074	963	set_transient('mxchat_admin_notice_error',
1075		- esc_html__('Failed to queue sitemap processing. Please check the status below for details.', 'mxchat'),
	964	+ esc_html__('Failed to start sitemap processing. Please check the status below for details.', 'mxchat'),
1076	965	30
1077	966	);
1078	967	}
1079	968
		@@ -1080,50 +969,127 @@
1080	969	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1081	970	exit;
1082	971	}
1083	972
1084		- // Handle Regular URL (single page)
	973	+ // Handle Regular URL
	974	+ //error_log('[MXCHAT-URL] Processing as regular webpage');
1085	975	$page_content = $this->mxchat_extract_main_content($body_content);
	976	+ //error_log('[MXCHAT-URL] Extracted content length: ' . strlen($page_content) . ' bytes');
	977	+
1086	978	$sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	979	+ //error_log('[MXCHAT-URL] Sanitized content length: ' . strlen($sanitized_content) . ' bytes');
1087	980
1088		- //error_log('[MXCHAT-URL-DEBUG] URL: ' . $submitted_url);
1089		- //error_log('[MXCHAT-URL-DEBUG] Extracted page_content length: ' . strlen($page_content));
1090		- //error_log('[MXCHAT-URL-DEBUG] Sanitized content length: ' . strlen($sanitized_content));
1091		- //error_log('[MXCHAT-URL-DEBUG] Content preview: ' . substr($sanitized_content, 0, 500));
1092		-
1093	981	if (empty($sanitized_content)) {
	982	+ //error_log('[MXCHAT-URL] Error: No valid content after sanitization');
	983	+
	984	+ // Set both transients - the error notice and the URL status
1094	985	set_transient('mxchat_admin_notice_error',
1095	986	esc_html__('No valid content found on the provided URL.', 'mxchat'),
1096	987	30
1097	988	);
	989	+
	990	+ // Set URL status transient
	991	+ set_transient('mxchat_single_url_status', [
	992	+ 'url' => $submitted_url,
	993	+ 'timestamp' => current_time('mysql'),
	994	+ 'status' => 'failed',
	995	+ 'error' => esc_html__('No valid content found on the provided URL.', 'mxchat')
	996	+ ], DAY_IN_SECONDS);
	997	+
1098	998	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1099	999	exit;
1100	1000	}
1101	1001
1102		- // For single URLs, process immediately using submit_content_to_db
1103		- // This handles chunking automatically for large content
1104		- $db_result = MxChat_Utils::submit_content_to_db(
1105		- $sanitized_content,
1106		- $submitted_url,
1107		- $api_key,
1108		- null,
1109		- $bot_id,
1110		- 'url' // content_type
1111		- );
	1002	+ //error_log('[MXCHAT-URL] Generating embedding for content');
	1003	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
1112	1004
1113		- if (is_wp_error($db_result)) {
1114		- $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1005	+ // Check if embedding_vector is a string (error message)
	1006	+ if (is_string($embedding_vector)) {
	1007	+ //error_log('[MXCHAT-URL] Error generating embedding: ' . $embedding_vector);
	1008	+ $error_message = esc_html__('Failed to generate embedding: ', 'mxchat') . esc_html($embedding_vector);
	1009	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1010	+
	1011	+ // Set both transients
1115	1012	set_transient('mxchat_admin_notice_error', $error_message, 30);
1116		- } else {
	1013	+
	1014	+ // Set URL status transient
	1015	+ set_transient('mxchat_single_url_status', [
	1016	+ 'url' => $submitted_url,
	1017	+ 'timestamp' => current_time('mysql'),
	1018	+ 'status' => 'failed',
	1019	+ 'error' => $error_message
	1020	+ ], DAY_IN_SECONDS);
	1021	+
	1022	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1023	+ exit;
	1024	+ }
	1025	+
	1026	+ if (is_array($embedding_vector)) {
	1027	+ //error_log('[MXCHAT-URL] Successfully generated embedding with ' . count($embedding_vector) . ' dimensions');
	1028	+
	1029	+ $db_result = MxChat_Utils::submit_content_to_db(
	1030	+ $sanitized_content,
	1031	+ $submitted_url,
	1032	+ $api_key
	1033	+ );
	1034	+
	1035	+ if (is_wp_error($db_result)) {
	1036	+ //error_log('[MXCHAT-URL] Error: Failed to store content in database: ' . $db_result->get_error_message());
	1037	+ $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1038	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1039	+
	1040	+ // Set both transients
	1041	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1042	+
	1043	+ // Set URL status transient
	1044	+ set_transient('mxchat_single_url_status', [
	1045	+ 'url' => $submitted_url,
	1046	+ 'timestamp' => current_time('mysql'),
	1047	+ 'status' => 'failed',
	1048	+ 'error' => $error_message
	1049	+ ], DAY_IN_SECONDS);
	1050	+
	1051	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1052	+ exit;
	1053	+ }
	1054	+
	1055	+ //error_log('[MXCHAT-URL] Successfully stored content in database');
1117	1056	$success_message = esc_html__('URL content successfully submitted!', 'mxchat');
	1057	+ //error_log('[MXCHAT-URL] Setting success transient: ' . $success_message);
	1058	+
	1059	+ // Set both transients
1118	1060	set_transient('mxchat_admin_notice_success', $success_message, 30);
	1061	+
	1062	+ // Set URL status transient with success
	1063	+ set_transient('mxchat_single_url_status', [
	1064	+ 'url' => $submitted_url,
	1065	+ 'timestamp' => current_time('mysql'),
	1066	+ 'status' => 'complete',
	1067	+ 'content_length' => strlen($sanitized_content),
	1068	+ 'embedding_dimensions' => count($embedding_vector)
	1069	+ ], DAY_IN_SECONDS);
	1070	+
	1071	+ } else {
	1072	+ //error_log('[MXCHAT-URL] Error: Failed to generate embedding. Unexpected result type: ' . gettype($embedding_vector));
	1073	+ $error_message = esc_html__('Failed to generate embedding: Unexpected result type. Please check your API key and try again.', 'mxchat');
	1074	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1075	+
	1076	+ // Set both transients
	1077	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1078	+
	1079	+ // Set URL status transient
	1080	+ set_transient('mxchat_single_url_status', [
	1081	+ 'url' => $submitted_url,
	1082	+ 'timestamp' => current_time('mysql'),
	1083	+ 'status' => 'failed',
	1084	+ 'error' => $error_message
	1085	+ ], DAY_IN_SECONDS);
1119	1086	}
1120	1087
	1088	+ //error_log('[MXCHAT-URL] ===== Completed URL submission process =====');
1121	1089	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1122	1090	exit;
1123	1091	}
1124		-
1125		-
1126	1092	public function mxchat_get_single_url_status() {
1127	1093	$status = get_transient('mxchat_single_url_status');
1128	1094	if (!$status) {
1129	1095	return null;
		@@ -1135,11 +1101,13 @@
1135	1101	}
1136	1102
1137	1103	return $status;
1138	1104	}
1139		-
1140		-public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url, $bot_id = 'default') {
	1105	+public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url) {
	1106	+ // Clear any single URL status when starting sitemap processing
	1107	+ delete_transient('mxchat_single_url_status');
1141	1108	if (!current_user_can('manage_options')) {
	1109	+ //error_log(esc_html__('Unauthorized sitemap processing attempt', 'mxchat'));
1142	1110	return false;
1143	1111	}
1144	1112
1145	1113	try {
		@@ -1148,96 +1116,331 @@
1148	1116	if (!$xml || !is_object($xml)) {
1149	1117	throw new Exception(__('Invalid XML object provided', 'mxchat'));
1150	1118	}
1151	1119
1152		- // Get bot-specific embedding API for validation
1153		- $bot_options = $this->get_bot_options($bot_id);
1154		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	1120	+ // Add embedding validation before processing
	1121	+ // Test embedding with a small sample text to verify API key is working
	1122	+ $test_result = $this->mxchat_generate_embedding("This is a test to verify the embedding API key is working.");
1155	1123
1156		- // Test the embedding API before processing
1157		- $test_phrase = "Test embedding generation for MxChat";
1158		- $test_result = $this->mxchat_generate_embedding($test_phrase, $bot_id);
1159		-
	1124	+ // Check if test_result is a string (error message) rather than an array (valid embedding)
1160	1125	if (is_string($test_result)) {
	1126	+ //error_log('[MXCHAT-URL] Embedding API validation failed: ' . $test_result);
	1127	+
	1128	+ // Store the error in the status transient so it can be displayed later
	1129	+ $status_data = array(
	1130	+ 'total_urls' => 0,
	1131	+ 'processed_urls' => 0,
	1132	+ 'status' => 'error',
	1133	+ 'error' => __('Embedding API validation failed: ', 'mxchat') . $test_result,
	1134	+ 'last_update' => time()
	1135	+ );
	1136	+
	1137	+ set_transient(
	1138	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1139	+ array_map('sanitize_text_field', $status_data),
	1140	+ DAY_IN_SECONDS
	1141	+ );
	1142	+
1161	1143	throw new Exception(__('Embedding API validation failed: ', 'mxchat') . $test_result);
1162	1144	}
1163	1145
	1146	+ // Make sure it's an array (valid embedding)
1164	1147	if (!is_array($test_result)) {
	1148	+ //error_log('[MXCHAT-URL] Embedding API returned unexpected result type: ' . gettype($test_result));
1165	1149	throw new Exception(__('Embedding API returned unexpected result type. Please check your configuration.', 'mxchat'));
1166	1150	}
1167	1151
1168		- // Extract URLs from sitemap
1169		- $urls = array();
	1152	+ $urls = [];
1170	1153	foreach ($xml->url as $url_element) {
1171	1154	$url = esc_url_raw((string)$url_element->loc);
1172	1155	if ($url) {
1173		- $urls[] = array('url' => $url);
	1156	+ $urls[] = $url;
1174	1157	}
1175	1158	}
1176	1159
1177		- $total_urls = count($urls);
	1160	+ $total_urls = absint(count($urls));
1178	1161
1179	1162	if ($total_urls < 1) {
1180	1163	throw new Exception(__('No valid URLs found in sitemap', 'mxchat'));
1181	1164	}
1182	1165
1183		- // Create unique queue ID
1184		- $queue_id = 'sitemap_' . md5($sitemap_url . time());
	1166	+ wp_schedule_single_event(time(), 'mxchat_process_sitemap_urls', array(
	1167	+ 'urls' => $urls,
	1168	+ 'sitemap_url' => $sitemap_url,
	1169	+ 'total_urls' => $total_urls,
	1170	+ 'batch_size' => absint(10),
	1171	+ 'batch_pause' => absint(5)
	1172	+ ));
1185	1173
1186		- // Add URLs to queue
1187		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'url', $urls, $bot_id);
	1174	+ $status_data = array(
	1175	+ 'total_urls' => $total_urls,
	1176	+ 'processed_urls' => 0,
	1177	+ 'status' => 'processing',
	1178	+ 'last_update' => time()
	1179	+ );
1188	1180
1189		- if ($queued_count === 0) {
1190		- throw new Exception(__('Failed to add URLs to processing queue', 'mxchat'));
1191		- }
	1181	+ set_transient(
	1182	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1183	+ array_map('sanitize_text_field', $status_data),
	1184	+ DAY_IN_SECONDS
	1185	+ );
1192	1186
1193		- // Store queue metadata
1194		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $sitemap_url);
1195		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'sitemap');
1196		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_urls);
1197		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
1198		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	1187	+ return __('scheduled', 'mxchat');
1199	1188
1200		- // Store queue ID in transient for status tracking
1201		- set_transient('mxchat_active_queue_sitemap', $queue_id, DAY_IN_SECONDS);
1202		- set_transient('mxchat_last_sitemap_url', $sitemap_url, DAY_IN_SECONDS);
1203		-
1204		- return 'queued';
1205		-
1206		- } catch (Exception $e) {
	1189	+ } catch (\Exception $e) {
1207	1190	$error_message = $e->getMessage();
1208	1191	//error_log(sprintf(esc_html__('Error preparing sitemap for processing: %s', 'mxchat'), esc_html($error_message)));
1209	1192
	1193	+ // Store the sitemap URL and error in transients so they can be displayed
	1194	+ set_transient(
	1195	+ 'mxchat_last_sitemap_url',
	1196	+ sanitize_text_field($sitemap_url),
	1197	+ DAY_IN_SECONDS
	1198	+ );
	1199	+
	1200	+ $status_data = array(
	1201	+ 'total_urls' => 0,
	1202	+ 'processed_urls' => 0,
	1203	+ 'status' => 'error',
	1204	+ 'error' => $error_message,
	1205	+ 'last_update' => time()
	1206	+ );
	1207	+
	1208	+ set_transient(
	1209	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1210	+ array_map('sanitize_text_field', $status_data),
	1211	+ DAY_IN_SECONDS
	1212	+ );
	1213	+
1210	1214	return $error_message;
1211	1215	}
1212		-
1213	1216	}
1214	1217
1215		-/**
1216		- * Remove shortcode tags but preserve the content inside them
1217		- * Example: [vc_column]Hello World[/vc_column] becomes "Hello World"
1218		- *
1219		- * @param string $content The content containing shortcodes
1220		- * @return string Content with shortcode tags removed but inner content preserved
1221		- */
1222		-private function strip_shortcode_tags_preserve_content($content) {
1223		- // Single-pass regex removes all shortcode brackets: [tag], [tag attr="val"], [/tag], [tag /]
1224		- // Content between tags is inherently preserved since only brackets are targeted
1225		- $result = preg_replace('/\[\/?\w[\w-]*[^\]]*\]/', '', $content);
1226		- return ($result !== null) ? $result : $content;
	1218	+public function mxchat_process_sitemap_urls_cron($urls, $sitemap_url, $total_urls, $batch_size, $batch_pause) {
	1219	+ // Validate inputs
	1220	+ $sitemap_url = esc_url_raw($sitemap_url);
	1221	+ $total_urls = absint($total_urls);
	1222	+ $batch_size = absint($batch_size);
	1223	+ $batch_pause = absint($batch_pause);
	1224	+
	1225	+ if (!is_array($urls) || empty($urls)) {
	1226	+ return;
	1227	+ }
	1228	+
	1229	+ try {
	1230	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	1231	+ $status = get_transient($status_key);
	1232	+
	1233	+ if (!$status || !is_array($status)) {
	1234	+ throw new Exception('Invalid status data retrieved from transient');
	1235	+ }
	1236	+
	1237	+ // Initialize failed_urls array if it doesn't exist
	1238	+ if (!isset($status['failed_urls_list']) || !is_array($status['failed_urls_list'])) {
	1239	+ $status['failed_urls_list'] = [];
	1240	+ }
	1241	+
	1242	+ $start_url = absint($status['processed_urls']);
	1243	+ $end_url = min($start_url + $batch_size, $total_urls);
	1244	+
	1245	+ // Track batch statistics
	1246	+ $batch_stats = [
	1247	+ 'processed' => 0,
	1248	+ 'failed' => 0,
	1249	+ 'last_error' => '',
	1250	+ 'embedding_errors' => 0
	1251	+ ];
	1252	+
	1253	+ // Check embedding configuration with first URL (only on first batch)
	1254	+ if ($start_url === 0) {
	1255	+ $test_url = esc_url_raw($urls[0]);
	1256	+ $test_response = wp_remote_get($test_url);
	1257	+
	1258	+ if (!is_wp_error($test_response) && wp_remote_retrieve_response_code($test_response) === 200) {
	1259	+ $test_html = wp_remote_retrieve_body($test_response);
	1260	+ $test_content = $this->mxchat_extract_main_content($test_html);
	1261	+ $test_sanitized = $this->mxchat_sanitize_content_for_api($test_content);
	1262	+
	1263	+ if (!empty($test_sanitized)) {
	1264	+ $test_embedding = $this->mxchat_generate_embedding($test_sanitized);
	1265	+
	1266	+ if (is_string($test_embedding)) {
	1267	+ throw new Exception('Embedding generation failed: ' . $test_embedding);
	1268	+ }
	1269	+
	1270	+ if (!is_array($test_embedding)) {
	1271	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($test_embedding));
	1272	+ }
	1273	+ }
	1274	+ }
	1275	+ }
	1276	+
	1277	+ for ($i = $start_url; $i < $end_url; $i++) {
	1278	+ $page_url = esc_url_raw($urls[$i]);
	1279	+ $max_retries = 3;
	1280	+ $retry_count = 0;
	1281	+ $url_processed = false;
	1282	+ $last_error = '';
	1283	+
	1284	+ while (!$url_processed && $retry_count < $max_retries) {
	1285	+ try {
	1286	+ // Attempt to fetch the URL
	1287	+ $page_response = wp_remote_get($page_url, array('timeout' => 30));
	1288	+
	1289	+ if (is_wp_error($page_response)) {
	1290	+ throw new Exception('HTTP request failed: ' . $page_response->get_error_message());
	1291	+ }
	1292	+
	1293	+ $response_code = wp_remote_retrieve_response_code($page_response);
	1294	+ if ($response_code !== 200) {
	1295	+ throw new Exception('HTTP Status: ' . $response_code);
	1296	+ }
	1297	+
	1298	+ $page_html = wp_remote_retrieve_body($page_response);
	1299	+
	1300	+ if (empty($page_html)) {
	1301	+ throw new Exception('Empty response body');
	1302	+ }
	1303	+
	1304	+ $page_content = $this->mxchat_extract_main_content($page_html);
	1305	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	1306	+
	1307	+ if (empty($sanitized_content)) {
	1308	+ throw new Exception('No valid content found after processing');
	1309	+ }
	1310	+
	1311	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	1312	+
	1313	+ if (is_string($embedding_vector)) {
	1314	+ throw new Exception('Embedding generation failed: ' . $embedding_vector);
	1315	+ }
	1316	+
	1317	+ if (!is_array($embedding_vector)) {
	1318	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($embedding_vector));
	1319	+ }
	1320	+
	1321	+ // Submit to database
	1322	+ $options = get_option('mxchat_options');
	1323	+ $submission_result = MxChat_Utils::submit_content_to_db($sanitized_content, $page_url, $options['api_key']);
	1324	+
	1325	+ if (is_wp_error($submission_result)) {
	1326	+ throw new Exception('Database submission failed: ' . $submission_result->get_error_message());
	1327	+ }
	1328	+
	1329	+ // Success!
	1330	+ $url_processed = true;
	1331	+ $batch_stats['processed']++;
	1332	+
	1333	+ } catch (Exception $e) {
	1334	+ $retry_count++;
	1335	+ $last_error = $e->getMessage();
	1336	+
	1337	+ //error_log("URL {$page_url} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	1338	+
	1339	+ // Track embedding errors specifically
	1340	+ if (strpos($last_error, 'Embedding') !== false) {
	1341	+ $batch_stats['embedding_errors']++;
	1342	+ }
	1343	+
	1344	+ if ($retry_count < $max_retries) {
	1345	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	1346	+ sleep(pow(2, $retry_count - 1));
	1347	+ }
	1348	+ }
	1349	+ }
	1350	+
	1351	+ // If URL still not processed after all retries, mark as failed
	1352	+ if (!$url_processed) {
	1353	+ $batch_stats['failed']++;
	1354	+ $batch_stats['last_error'] = $last_error;
	1355	+
	1356	+ // Add to failed URLs list
	1357	+ $status['failed_urls_list'][] = [
	1358	+ 'url' => $page_url,
	1359	+ 'error' => $last_error,
	1360	+ 'time' => time(),
	1361	+ 'retries' => $max_retries
	1362	+ ];
	1363	+
	1364	+ // Limit the number of failed URLs we store to prevent transient size issues
	1365	+ if (count($status['failed_urls_list']) > 100) {
	1366	+ $status['failed_urls_list'] = array_slice($status['failed_urls_list'], -100);
	1367	+ }
	1368	+ }
	1369	+
	1370	+ // Update progress
	1371	+ $status['processed_urls'] = absint($i + 1);
	1372	+ $status['last_update'] = time();
	1373	+ $status['failed_urls'] = absint($status['failed_urls'] ?? 0) + ($url_processed ? 0 : 1);
	1374	+ $status['last_error'] = $batch_stats['last_error'];
	1375	+
	1376	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1377	+
	1378	+ // If we have too many consecutive embedding errors, stop processing
	1379	+ if ($batch_stats['embedding_errors'] >= 10) {
	1380	+ throw new Exception('Too many consecutive embedding failures detected. Please check your API configuration.');
	1381	+ }
	1382	+ }
	1383	+
	1384	+ // If all URLs in this batch failed, stop processing
	1385	+ if ($batch_stats['processed'] === 0 && $batch_stats['failed'] > 0) {
	1386	+ $status['status'] = 'error';
	1387	+ $status['error'] = sprintf(
	1388	+ 'Processing stopped: %d consecutive failures in batch. Last error: %s',
	1389	+ $batch_stats['failed'],
	1390	+ $batch_stats['last_error']
	1391	+ );
	1392	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1393	+ return;
	1394	+ }
	1395	+
	1396	+ // Update final progress
	1397	+ $status['processed_urls'] = min($end_url, $total_urls);
	1398	+ $status['last_update'] = time();
	1399	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1400	+
	1401	+ // Check if we've processed all URLs
	1402	+ if ($end_url >= $total_urls) {
	1403	+ // All URLs have been processed - mark as complete
	1404	+ $status['status'] = 'complete';
	1405	+ $status['processed_urls'] = $total_urls;
	1406	+
	1407	+ // Add completion summary
	1408	+ $status['completion_summary'] = [
	1409	+ 'total_urls' => $total_urls,
	1410	+ 'successful_urls' => $total_urls - absint($status['failed_urls'] ?? 0),
	1411	+ 'failed_urls' => absint($status['failed_urls'] ?? 0),
	1412	+ 'completion_time' => current_time('mysql')
	1413	+ ];
	1414	+
	1415	+ // Save the completed status (don't delete it - let user dismiss manually)
	1416	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1417	+
	1418	+ // DON'T delete the status transients here - let user dismiss manually
	1419	+ } else {
	1420	+ // Schedule next batch
	1421	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_sitemap_urls', array(
	1422	+ 'urls' => $urls,
	1423	+ 'sitemap_url' => $sitemap_url,
	1424	+ 'total_urls' => $total_urls,
	1425	+ 'batch_size' => $batch_size,
	1426	+ 'batch_pause' => $batch_pause,
	1427	+ ));
	1428	+ }
	1429	+ } catch (\Exception $e) {
	1430	+ $status['status'] = 'error';
	1431	+ $status['error'] = $e->getMessage();
	1432	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1433	+ }
1227	1434	}
1228		-
1229	1435	public function mxchat_sanitize_content_for_api($content) {
1230	1436	//error_log('[MXCHAT-SANITIZE] Original content preview: ' . substr($content, 0, 500) . '...');
1231		-
1232		- // Remove shortcode tags but PRESERVE content inside them
1233		- $content = $this->strip_shortcode_tags_preserve_content($content);
1234		-
	1437	+
1235	1438	// Remove script, style tags, and HTML comments
1236	1439	$content = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $content);
1237	1440	$content = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $content);
1238	1441	$content = preg_replace('/<!--(.|\s)*?-->/', '', $content);
1239		-
	1442	+
1240	1443	// Remove all HTML tags and decode HTML entities
1241	1444	$content = wp_strip_all_tags($content);
1242	1445	$content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5);
1243	1446
		@@ -1295,12 +1498,12 @@
1295	1498	@$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'), LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
1296	1499	$xpath = new DOMXPath($dom);
1297	1500
1298	1501	// For debugging purposes
1299		- $debugEnabled = true; // Set to true to enable debugging output
	1502	+ $debugEnabled = false; // Set to true to enable debugging output
1300	1503	$debug = function($message) use ($debugEnabled) {
1301	1504	if ($debugEnabled) {
1302		- //error_log('[MXCHAT-EXTRACT-DEBUG] ' . $message);
	1505	+ //error_log('[MXCHAT-DEBUG] ' . $message);
1303	1506	}
1304	1507	};
1305	1508
1306	1509	// Direct targeting for Gerow theme posts
		@@ -1399,25 +1602,18 @@
1399	1602	'//main',
1400	1603	'//div[contains(@class, "content")]'
1401	1604	];
1402	1605
1403		- // First handle Elementor content - get only leaf widget containers to avoid duplicates
	1606	+ // First handle Elementor content
1404	1607	$debug("Checking for Elementor content");
1405		- // Get widget containers that are direct children of widgets (not nested inside other widget containers)
1406		- $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-widget")]//div[contains(@class, "elementor-widget-container")]');
	1608	+ $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-element")]//div[contains(@class, "elementor-widget-container")]');
1407	1609	if ($elementor_widgets && $elementor_widgets->length > 0) {
1408	1610	$debug("Found Elementor widgets");
1409		- $seen_content = array(); // Track seen content to avoid duplicates
1410	1611	$combined_content = '';
1411	1612	foreach ($elementor_widgets as $widget) {
1412	1613	$widget_content = $dom->saveHTML($widget);
1413	1614	if (!empty($widget_content)) {
1414		- // Create a hash of the content to detect duplicates
1415		- $content_hash = md5($widget_content);
1416		- if (!isset($seen_content[$content_hash])) {
1417		- $seen_content[$content_hash] = true;
1418		- $combined_content .= $widget_content;
1419		- }
	1615	+ $combined_content .= $widget_content;
1420	1616	}
1421	1617	}
1422	1618	if (!empty($combined_content)) {
1423	1619	$debug("Returning Elementor content");
		@@ -1429,14 +1625,15 @@
1429	1625	foreach ($selectors as $selector) {
1430	1626	$debug("Trying selector: " . $selector);
1431	1627	$nodes = $xpath->query($selector);
1432	1628	if ($nodes && $nodes->length > 0) {
1433		- $debug("Found " . $nodes->length . " matches for selector: " . $selector);
1434		- // Only take the FIRST matching node to avoid duplicate content
1435		- // (pages often have nested or multiple containers with same class)
1436		- $content = $dom->saveHTML($nodes->item(0));
	1629	+ $debug("Found matches for selector: " . $selector);
	1630	+ $content = '';
	1631	+ foreach ($nodes as $node) {
	1632	+ $content .= $dom->saveHTML($node);
	1633	+ }
1437	1634	if (!empty($content)) {
1438		- $debug("Returning content from selector: " . $selector . " (first match only)");
	1635	+ $debug("Returning content from selector: " . $selector);
1439	1636	return $content;
1440	1637	}
1441	1638	}
1442	1639	}
		@@ -1460,106 +1657,16 @@
1460	1657	$debug("Returning blog-area section content");
1461	1658	return $content;
1462	1659	}
1463	1660	}
1464		-
1465		- // Generic container selectors for non-CMS sites (like .asp pages)
1466		- $debug("Trying generic container selectors");
1467		- $generic_selectors = [
1468		- '//div[@id="main"]',
1469		- '//div[@id="wrapper"]',
1470		- '//div[@id="page"]',
1471		- '//div[@id="site-content"]',
1472		- '//div[contains(@class, "main-content")]',
1473		- '//div[contains(@class, "page-content")]',
1474		- '//div[contains(@class, "site-content")]',
1475		- ];
1476		-
1477		- foreach ($generic_selectors as $selector) {
1478		- $debug("Trying generic selector: " . $selector);
1479		- $nodes = $xpath->query($selector);
1480		- if ($nodes && $nodes->length > 0) {
1481		- $content = $dom->saveHTML($nodes->item(0));
1482		- if (!empty($content)) {
1483		- $debug("Returning content from generic selector: " . $selector);
1484		- return $content;
1485		- }
1486		- }
1487		- }
1488		-
1489		- // Paragraph-based content detection - find regions with substantial text
1490		- $debug("Trying paragraph-based content detection");
1491		- $paragraphs = $xpath->query('//p[string-length(normalize-space()) > 50]');
1492		- if ($paragraphs && $paragraphs->length >= 3) {
1493		- $debug("Found " . $paragraphs->length . " substantial paragraphs");
1494		- // Collect all substantial paragraphs and their content
1495		- $paragraph_content = '';
1496		- foreach ($paragraphs as $p) {
1497		- $paragraph_content .= $dom->saveHTML($p) . "\n";
1498		- }
1499		- if (!empty($paragraph_content)) {
1500		- $debug("Returning paragraph-based content");
1501		- return $paragraph_content;
1502		- }
1503		- }
1504		-
1505		- // Improved body fallback - strip nav/header/footer elements first
1506		- $debug("Using improved body fallback");
	1661	+
	1662	+ // Fallback: Return the body content if no specific selector matches
	1663	+ $debug("Using body fallback");
1507	1664	$body = $dom->getElementsByTagName('body');
1508	1665	if ($body->length > 0) {
1509		- // Clone the body to avoid modifying the original DOM
1510		- $body_clone = $body->item(0)->cloneNode(true);
1511		-
1512		- // Remove common non-content elements by tag name
1513		- $remove_tags = ['nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript'];
1514		- foreach ($remove_tags as $tag) {
1515		- $elements = $body_clone->getElementsByTagName($tag);
1516		- // Iterate backwards to safely remove elements
1517		- for ($i = $elements->length - 1; $i >= 0; $i--) {
1518		- $el = $elements->item($i);
1519		- if ($el && $el->parentNode) {
1520		- $el->parentNode->removeChild($el);
1521		- }
1522		- }
1523		- }
1524		-
1525		- // Remove elements with common non-content class names using XPath on the cloned body
1526		- $temp_dom = new DOMDocument();
1527		- @$temp_dom->appendChild($temp_dom->importNode($body_clone, true));
1528		- $temp_xpath = new DOMXPath($temp_dom);
1529		-
1530		- $remove_class_patterns = [
1531		- '//*[contains(@class, "nav")]',
1532		- '//*[contains(@class, "menu")]',
1533		- '//*[contains(@class, "sidebar")]',
1534		- '//*[contains(@class, "footer")]',
1535		- '//*[contains(@class, "header")]',
1536		- '//*[contains(@id, "nav")]',
1537		- '//*[contains(@id, "menu")]',
1538		- '//*[contains(@id, "sidebar")]',
1539		- '//*[contains(@id, "footer")]',
1540		- '//*[contains(@id, "header")]',
1541		- ];
1542		-
1543		- foreach ($remove_class_patterns as $pattern) {
1544		- $elements = $temp_xpath->query($pattern);
1545		- if ($elements) {
1546		- for ($i = $elements->length - 1; $i >= 0; $i--) {
1547		- $el = $elements->item($i);
1548		- if ($el && $el->parentNode) {
1549		- $el->parentNode->removeChild($el);
1550		- }
1551		- }
1552		- }
1553		- }
1554		-
1555		- $cleaned_content = $temp_dom->saveHTML();
1556		- if (!empty($cleaned_content)) {
1557		- $debug("Returning cleaned body content");
1558		- return $cleaned_content;
1559		- }
	1666	+ return $dom->saveHTML($body->item(0));
1560	1667	}
1561		-
	1668	+
1562	1669	// Last resort: return the original HTML
1563	1670	$debug("Returning original HTML");
1564	1671	return $html;
1565	1672	} catch (Exception $e) {
		@@ -1611,41 +1718,47 @@
1611	1718	try {
1612	1719	// Verify the request
1613	1720	check_ajax_referer('mxchat_status_nonce', 'nonce');
1614	1721
1615		- // Get active queue IDs
1616		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
1617		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
	1722	+ // Get the status just like in your admin page
	1723	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1724	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
1618	1725
1619		- $sitemap_status = false;
1620		- $pdf_status = false;
	1726	+ $pdf_status = $pdf_url ? $this->mxchat_get_pdf_processing_status($pdf_url) : false;
	1727	+ $sitemap_status = $sitemap_url ? $this->mxchat_get_sitemap_processing_status($sitemap_url) : false;
1621	1728
1622		- // Get sitemap queue status
1623		- if ($sitemap_queue_id) {
1624		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
	1729	+ // Add the PDF URL to the status object
	1730	+ if ($pdf_status && $pdf_url) {
	1731	+ $pdf_status['pdf_url'] = $pdf_url;
1625	1732	}
1626	1733
1627		- // Get PDF queue status
1628		- if ($pdf_queue_id) {
1629		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
1630		- }
	1734	+ // Set the current PDF URL for the manual batch processing button
	1735	+ $current_pdf_url = $pdf_url;
1631	1736
	1737	+ // Check for true processing status, not just presence of status
1632	1738	$is_active_processing =
1633		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
1634		- ($pdf_status && $pdf_status['status'] === 'processing');
	1739	+ ($sitemap_status && isset($sitemap_status['status']) && $sitemap_status['status'] === 'processing') ||
	1740	+ ($pdf_status && isset($pdf_status['status']) && $pdf_status['status'] === 'processing');
1635	1741
	1742	+ // Get single URL status, but only if no processing is active
	1743	+ $single_url_status = !$is_active_processing ? $this->mxchat_get_single_url_status() : false;
	1744	+
	1745	+ // REMOVED: Auto-clearing of completed status - now only done via dismiss button
	1746	+
1636	1747	// Return JSON response with the status data
1637	1748	wp_send_json(array(
1638	1749	'pdf_status' => $pdf_status,
1639	1750	'sitemap_status' => $sitemap_status,
	1751	+ 'single_url_status' => $single_url_status,
1640	1752	'is_processing' => $is_active_processing,
1641		- 'sitemap_queue_id' => $sitemap_queue_id,
1642		- 'pdf_queue_id' => $pdf_queue_id
	1753	+ 'current_pdf_url' => $current_pdf_url
1643	1754	));
1644	1755
1645	1756	} catch (Exception $e) {
	1757	+ // Log the error
1646	1758	//error_log('MxChat Status Update Error: ' . $e->getMessage());
1647	1759
	1760	+ // Return a friendly error response
1648	1761	wp_send_json_error(array(
1649	1762	'message' => 'Error getting status updates: ' . $e->getMessage(),
1650	1763	'status' => 'error'
1651	1764	));
		@@ -1650,1373 +1763,8 @@
1650	1763	'status' => 'error'
1651	1764	));
1652	1765	}
1653	1766	}
1654		-
1655		-/**
1656		- * Helper function to get queue status data
1657		- */
1658		-private function mxchat_get_queue_status_data($queue_id, $type = 'sitemap') {
1659		- global $wpdb;
1660		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
1661		-
1662		- // Get counts by status
1663		- $counts = $wpdb->get_results($wpdb->prepare(
1664		- "SELECT status, COUNT(*) as count
1665		- FROM $table_name
1666		- WHERE queue_id = %s
1667		- GROUP BY status",
1668		- $queue_id
1669		- ), OBJECT_K);
1670		-
1671		- $total = 0;
1672		- $completed = 0;
1673		- $failed = 0;
1674		- $processing = 0;
1675		- $pending = 0;
1676		-
1677		- foreach ($counts as $status => $data) {
1678		- $count = absint($data->count);
1679		- $total += $count;
1680		-
1681		- switch ($status) {
1682		- case 'completed':
1683		- $completed = $count;
1684		- break;
1685		- case 'failed':
1686		- $failed = $count;
1687		- break;
1688		- case 'processing':
1689		- $processing = $count;
1690		- break;
1691		- case 'pending':
1692		- $pending = $count;
1693		- break;
1694		- }
1695		- }
1696		-
1697		- if ($total === 0) {
1698		- return false;
1699		- }
1700		-
1701		- // Calculate percentage
1702		- $percentage = round((($completed + $failed) / $total) * 100);
1703		-
1704		- // Get failed items details (limit to 50)
1705		- $failed_items = array();
1706		- if ($failed > 0) {
1707		- $failed_results = $wpdb->get_results($wpdb->prepare(
1708		- "SELECT item_type, item_data, error_message, attempts, completed_at
1709		- FROM $table_name
1710		- WHERE queue_id = %s
1711		- AND status = 'failed'
1712		- AND attempts >= max_attempts
1713		- ORDER BY id DESC
1714		- LIMIT 50",
1715		- $queue_id
1716		- ));
1717		-
1718		- foreach ($failed_results as $item) {
1719		- $data = json_decode($item->item_data, true);
1720		- $url = $type === 'pdf' ? ($data['pdf_url'] ?? '') : ($data['url'] ?? '');
1721		-
1722		- $failed_items[] = array(
1723		- 'url' => $url,
1724		- 'page' => $type === 'pdf' ? ($data['page_number'] ?? 0) : 0,
1725		- 'error' => $item->error_message,
1726		- 'retries' => $item->attempts,
1727		- 'time' => strtotime($item->completed_at)
1728		- );
1729		- }
1730		- }
1731		-
1732		- // Get queue metadata
1733		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
1734		-
1735		- // Determine if queue is complete
1736		- $is_complete = ($pending === 0 && $processing === 0);
1737		-
1738		- // Get last update time
1739		- $last_update = $wpdb->get_var($wpdb->prepare(
1740		- "SELECT MAX(UNIX_TIMESTAMP(COALESCE(completed_at, started_at, created_at)))
1741		- FROM $table_name
1742		- WHERE queue_id = %s",
1743		- $queue_id
1744		- ));
1745		-
1746		- $last_update_text = $last_update ? human_time_diff($last_update, time()) . ' ' . __('ago', 'mxchat') : __('Just now', 'mxchat');
1747		-
1748		- // Format based on type
1749		- if ($type === 'pdf') {
1750		- return array(
1751		- 'total_pages' => $total,
1752		- 'processed_pages' => $completed + $failed,
1753		- 'failed_pages' => $failed,
1754		- 'percentage' => $percentage,
1755		- 'status' => $is_complete ? 'complete' : 'processing',
1756		- 'last_update' => $last_update_text,
1757		- 'failed_pages_list' => $failed_items,
1758		- 'pdf_url' => $source_url,
1759		- 'queue_id' => $queue_id
1760		- );
1761		- } else {
1762		- return array(
1763		- 'total_urls' => $total,
1764		- 'processed_urls' => $completed + $failed,
1765		- 'failed_urls' => $failed,
1766		- 'percentage' => $percentage,
1767		- 'status' => $is_complete ? 'complete' : 'processing',
1768		- 'last_update' => $last_update_text,
1769		- 'failed_urls_list' => $failed_items,
1770		- 'sitemap_url' => $source_url,
1771		- 'queue_id' => $queue_id
1772		- );
1773		- }
1774		-}
1775		-
1776		-/**
1777		- * Public method to get processing status for both sitemap and PDF queues
1778		- * Used by admin pages to display processing status
1779		- *
1780		- * @return array Array with 'sitemap_status', 'pdf_status', and 'is_processing' keys
1781		- */
1782		-public function mxchat_get_processing_statuses() {
1783		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
1784		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
1785		-
1786		- $sitemap_status = false;
1787		- $pdf_status = false;
1788		-
1789		- if ($sitemap_queue_id) {
1790		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
1791		- }
1792		-
1793		- if ($pdf_queue_id) {
1794		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
1795		- }
1796		-
1797		- $is_processing =
1798		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
1799		- ($pdf_status && $pdf_status['status'] === 'processing');
1800		-
1801		- return array(
1802		- 'sitemap_status' => $sitemap_status,
1803		- 'pdf_status' => $pdf_status,
1804		- 'is_processing' => $is_processing
1805		- );
1806		-}
1807		-
1808		-/**
1809		- * AJAX handler to get recent knowledge entries for real-time table updates
1810		- * UPDATED: Now supports both WordPress DB and Pinecone data sources
1811		- */
1812		-public function ajax_mxchat_get_recent_entries() {
1813		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
1814		-
1815		- if (!current_user_can('manage_options')) {
1816		- wp_send_json_error(array('message' => 'Unauthorized'));
1817		- return;
1818		- }
1819		-
1820		- global $wpdb;
1821		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1822		-
1823		- // Get parameters
1824		- $last_id = isset($_POST['last_id']) ? absint($_POST['last_id']) : 0;
1825		- $limit = isset($_POST['limit']) ? min(absint($_POST['limit']), 50) : 10;
1826		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
1827		-
1828		- // Check if Pinecone is enabled for this bot
1829		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
1830		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
1831		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
1832		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
1833		-
1834		- if ($use_pinecone && $has_pinecone_api) {
1835		- // PINECONE DATA SOURCE - Get grouped entry count (not raw vector count)
1836		- // Use mxchat_fetch_pinecone_records which returns total_unique_entries
1837		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, '', 1, 10, $bot_id, '');
1838		- $total_count = $records['total'] ?? 0;
1839		-
1840		- // For Pinecone, we don't return individual entries during polling
1841		- // (entries are already displayed on page load via mxchat_fetch_pinecone_records)
1842		- // We just return the updated count
1843		- wp_send_json_success(array(
1844		- 'entries' => array(),
1845		- 'total_count' => absint($total_count),
1846		- 'max_id' => $last_id,
1847		- 'data_source' => 'pinecone'
1848		- ));
1849		- return;
1850		- }
1851		-
1852		- // WORDPRESS DB DATA SOURCE
1853		- // Build query to get entries newer than last_id
1854		- $where_clauses = array('1=1');
1855		- $where_values = array();
1856		-
1857		- if ($last_id > 0) {
1858		- $where_clauses[] = 'id > %d';
1859		- $where_values[] = $last_id;
1860		- }
1861		-
1862		- // Note: WordPress DB table doesn't have bot_id column
1863		- // Multi-bot filtering is handled via Pinecone namespaces
1864		-
1865		- $where_sql = implode(' AND ', $where_clauses);
1866		-
1867		- // Get recent entries
1868		- $query = "SELECT id, article_content, source_url, timestamp
1869		- FROM $table_name
1870		- WHERE $where_sql
1871		- ORDER BY id DESC
1872		- LIMIT %d";
1873		-
1874		- $where_values[] = $limit;
1875		-
1876		- $entries = $wpdb->get_results($wpdb->prepare($query, $where_values));
1877		-
1878		- // Get total count of GROUPED entries (by source_url) - matches pagination display
1879		- // Count unique source_urls (excluding mxchat:// internal refs) + count of ungrouped rows
1880		- $total_count = $wpdb->get_var(
1881		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
1882		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
1883		- );
1884		-
1885		- // Format entries for response
1886		- $formatted_entries = array();
1887		- $preview_length = 150;
1888		- foreach ($entries as $entry) {
1889		- // Parse chunk metadata using the proper chunker method (same as initial page load)
1890		- if (class_exists('MxChat_Chunker')) {
1891		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($entry->article_content);
1892		- $display_content = $chunk_meta['text'];
1893		- $chunk_metadata = $chunk_meta['metadata'];
1894		- } else {
1895		- $display_content = $entry->article_content;
1896		- $chunk_metadata = array();
1897		- }
1898		-
1899		- $content_preview = mb_strlen($display_content) > $preview_length
1900		- ? mb_substr($display_content, 0, $preview_length) . '...'
1901		- : $display_content;
1902		-
1903		- $formatted_entries[] = array(
1904		- 'id' => $entry->id,
1905		- 'preview' => esc_html($content_preview),
1906		- 'full_content' => wp_kses_post(wpautop($display_content)),
1907		- 'content_length' => mb_strlen($display_content),
1908		- 'preview_length' => $preview_length,
1909		- 'source_url' => $entry->source_url,
1910		- 'has_link' => !empty($entry->source_url) && strpos($entry->source_url, 'mxchat://') !== 0,
1911		- 'chunk_metadata' => $chunk_metadata,
1912		- 'bot_id' => $entry->bot_id ?? 'default',
1913		- 'edit_nonce' => wp_create_nonce('mxchat_edit_entry_nonce'),
1914		- 'delete_nonce' => wp_create_nonce('mxchat_delete_prompt_nonce')
1915		- );
1916		- }
1917		-
1918		- wp_send_json_success(array(
1919		- 'entries' => $formatted_entries,
1920		- 'total_count' => absint($total_count),
1921		- 'max_id' => !empty($entries) ? $entries[0]->id : $last_id,
1922		- 'data_source' => 'wordpress'
1923		- ));
1924		-}
1925		-
1926		-/**
1927		- * Get Pinecone total count from stats API
1928		- * Helper function for ajax_mxchat_get_recent_entries
1929		- */
1930		-private function mxchat_get_pinecone_count_from_stats($pinecone_options) {
1931		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1932		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1933		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1934		-
1935		- if (empty($api_key) || empty($host)) {
1936		- return 0;
1937		- }
1938		-
1939		- try {
1940		- $stats_url = "https://{$host}/describe_index_stats";
1941		-
1942		- $response = wp_remote_post($stats_url, array(
1943		- 'headers' => array(
1944		- 'Api-Key' => $api_key,
1945		- 'Content-Type' => 'application/json'
1946		- ),
1947		- 'body' => '{}',
1948		- 'timeout' => 10
1949		- ));
1950		-
1951		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
1952		- $body = wp_remote_retrieve_body($response);
1953		- $stats_data = json_decode($body, true);
1954		-
1955		- // If namespace is specified, get count from that specific namespace
1956		- if (!empty($namespace) && isset($stats_data['namespaces'][$namespace]['vectorCount'])) {
1957		- return intval($stats_data['namespaces'][$namespace]['vectorCount']);
1958		- }
1959		-
1960		- // If no namespace specified or namespace not found in response, use total
1961		- return intval($stats_data['totalVectorCount'] ?? 0);
1962		- }
1963		-
1964		- return 0;
1965		-
1966		- } catch (Exception $e) {
1967		- return 0;
1968		- }
1969		-}
1970		-
1971		-/**
1972		- * AJAX handler to refresh Pinecone entries table via AJAX
1973		- * Returns the table HTML for updating the UI without a full page reload
1974		- */
1975		-public function ajax_mxchat_refresh_pinecone_entries() {
1976		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
1977		-
1978		- if (!current_user_can('manage_options')) {
1979		- wp_send_json_error(array('message' => 'Unauthorized'));
1980		- return;
1981		- }
1982		-
1983		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
1984		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
1985		- $per_page = 25;
1986		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
1987		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
1988		-
1989		- // Get Pinecone manager and options
1990		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
1991		- if (!$pinecone_manager) {
1992		- wp_send_json_error(array('message' => 'Pinecone manager not available'));
1993		- return;
1994		- }
1995		-
1996		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
1997		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
1998		- $pinecone_api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1999		-
2000		- if (!$use_pinecone || empty($pinecone_api_key)) {
2001		- wp_send_json_error(array('message' => 'Pinecone not configured'));
2002		- return;
2003		- }
2004		-
2005		- // Fetch records from Pinecone
2006		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, $search_query, $page, $per_page, $bot_id, $content_type_filter);
2007		- $prompts = $records['data'] ?? array();
2008		- $total_records = $records['total'] ?? 0;
2009		-
2010		- // Preprocess Pinecone records — set chunk_metadata and display_content
2011		- // (matches admin-knowledge-page.php preprocessing)
2012		- foreach ($prompts as $prompt) {
2013		- if (isset($prompt->chunk_index) && $prompt->chunk_index !== null) {
2014		- $prompt->chunk_metadata = array(
2015		- 'chunk_index' => intval($prompt->chunk_index),
2016		- 'total_chunks' => isset($prompt->total_chunks) ? intval($prompt->total_chunks) : null,
2017		- 'is_chunked' => isset($prompt->is_chunked) ? (bool) $prompt->is_chunked : true,
2018		- 'source_url' => $prompt->source_url ?? ''
2019		- );
2020		- $prompt->display_content = $prompt->article_content;
2021		- } else {
2022		- $prompt->chunk_metadata = array();
2023		- $prompt->display_content = $prompt->article_content ?? '';
2024		- }
2025		- }
2026		-
2027		- // Group prompts by source_url
2028		- $grouped_prompts = array();
2029		- foreach ($prompts as $prompt) {
2030		- $source_url = '';
2031		- if (!empty($prompt->chunk_metadata['source_url'])) {
2032		- $source_url = $prompt->chunk_metadata['source_url'];
2033		- } elseif (!empty($prompt->source_url)) {
2034		- $source_url = $prompt->source_url;
2035		- }
2036		-
2037		- if (!empty($source_url)) {
2038		- if (!isset($grouped_prompts[$source_url])) {
2039		- $grouped_prompts[$source_url] = array();
2040		- }
2041		- $grouped_prompts[$source_url][] = $prompt;
2042		- } else {
2043		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2044		- }
2045		- }
2046		-
2047		- // Sort each group by chunk_index
2048		- foreach ($grouped_prompts as $source_url => &$group) {
2049		- usort($group, function($a, $b) {
2050		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2051		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2052		- return $index_a - $index_b;
2053		- });
2054		- }
2055		- unset($group);
2056		-
2057		- // Build HTML for the table rows - must match admin-knowledge-page.php structure exactly
2058		- ob_start();
2059		- $display_index = 0;
2060		- $current_page = $page;
2061		- $data_source = 'pinecone';
2062		- $current_bot_id = $bot_id;
2063		- $preview_length = 150;
2064		-
2065		- if (empty($grouped_prompts)) {
2066		- echo '<tr><td colspan="4" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2067		- esc_html_e('No knowledge entries found in Pinecone.', 'mxchat');
2068		- echo '</td></tr>';
2069		- } else {
2070		- foreach ($grouped_prompts as $source_url => $group) {
2071		- $chunk_count = count($group);
2072		- $first_prompt = $group[0];
2073		- $display_index++;
2074		-
2075		- if ($chunk_count > 1) {
2076		- // Multiple chunks - show grouped row with expand button
2077		- $group_id = 'group-' . md5($source_url);
2078		- ?>
2079		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2080		- class="mxchat-chunk-group-header"
2081		- data-source="<?php echo esc_attr($data_source); ?>"
2082		- data-group-id="<?php echo esc_attr($group_id); ?>"
2083		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2084		- <td style="padding: 12px 16px; text-align: center;">
2085		- <input type="checkbox"
2086		- class="mxchat-entry-checkbox"
2087		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2088		- data-source="<?php echo esc_attr($data_source); ?>"
2089		- data-source-url="<?php echo esc_attr($source_url); ?>"
2090		- data-is-group="true"
2091		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2092		- </td>
2093		- <td style="padding: 12px 16px; font-size: 13px;">
2094		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2095		- </td>
2096		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2097		- <div class="mxchat-chunk-group-info">
2098		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2099		- <span class="dashicons dashicons-arrow-right-alt2"></span>
2100		- </button>
2101		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2102		- <span class="mxchat-chunk-preview">
2103		- <?php
2104		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : (isset($first_prompt->article_content) ? $first_prompt->article_content : '');
2105		- $content_preview = mb_substr($parent_content, 0, 100);
2106		- echo esc_html($content_preview . '...');
2107		- ?>
2108		- </span>
2109		- </div>
2110		- </td>
2111		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2112		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2113		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2114		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2115		- <?php esc_html_e('View Source', 'mxchat'); ?>
2116		- </a>
2117		- <?php else : ?>
2118		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2119		- <?php endif; ?>
2120		- </td>
2121		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2122		- <?php if ($data_source !== 'pinecone') : ?>
2123		- <button type="button"
2124		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2125		- data-source-url="<?php echo esc_attr($source_url); ?>"
2126		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2127		- data-data-source="<?php echo esc_attr($data_source); ?>"
2128		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2129		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2130		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2131		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2132		- </button>
2133		- <?php endif; ?>
2134		- <button type="button"
2135		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2136		- data-source-url="<?php echo esc_attr($source_url); ?>"
2137		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2138		- data-data-source="<?php echo esc_attr($data_source); ?>"
2139		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2140		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2141		- style="color: var(--mxch-error);"
2142		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2143		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2144		- </button>
2145		- </td>
2146		- </tr>
2147		- <?php
2148		- // Render hidden chunk rows
2149		- foreach ($group as $chunk_index => $chunk) {
2150		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
2151		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
2152		- $content = isset($chunk->display_content) ? $chunk->display_content : (isset($chunk->article_content) ? $chunk->article_content : '');
2153		- $content_preview = mb_strlen($content) > $preview_length
2154		- ? mb_substr($content, 0, $preview_length) . '...'
2155		- : $content;
2156		- ?>
2157		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
2158		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
2159		- data-source="<?php echo esc_attr($data_source); ?>"
2160		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
2161		- <td style="padding: 12px 16px; text-align: center;">
2162		- <!-- Checkbox column placeholder for chunks (managed by group) -->
2163		- </td>
2164		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
2165		- <!-- Hidden ID column for chunks -->
2166		- </td>
2167		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2168		- <div class="mxchat-accordion-wrapper">
2169		- <div class="mxchat-content-preview">
2170		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
2171		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
2172		- </span>
2173		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2174		- <?php if (mb_strlen($content) > $preview_length) : ?>
2175		- <button class="mxchat-expand-toggle" type="button">
2176		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2177		- </button>
2178		- <?php endif; ?>
2179		- </div>
2180		- <div class="mxchat-content-full" style="display: none;">
2181		- <div class="content-view">
2182		- <?php
2183		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2184		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2185		- echo wp_kses_post(wpautop($content));
2186		- echo '</div>';
2187		- } else {
2188		- echo wp_kses_post(wpautop($content));
2189		- }
2190		- ?>
2191		- </div>
2192		- </div>
2193		- </div>
2194		- </td>
2195		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2196		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
2197		- </td>
2198		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
2199		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
2200		- </td>
2201		- </tr>
2202		- <?php
2203		- }
2204		- } else {
2205		- // Single entry - display normally with accordion
2206		- $prompt = $first_prompt;
2207		- $content = isset($prompt->display_content) ? $prompt->display_content : (isset($prompt->article_content) ? $prompt->article_content : '');
2208		- $content_preview = mb_strlen($content) > $preview_length
2209		- ? mb_substr($content, 0, $preview_length) . '...'
2210		- : $content;
2211		- ?>
2212		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
2213		- data-source="<?php echo esc_attr($data_source); ?>"
2214		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2215		- <td style="padding: 12px 16px; text-align: center;">
2216		- <input type="checkbox"
2217		- class="mxchat-entry-checkbox"
2218		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2219		- data-source="<?php echo esc_attr($data_source); ?>"
2220		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
2221		- data-is-group="false"
2222		- data-chunk-count="1">
2223		- </td>
2224		- <td style="padding: 12px 16px; font-size: 13px;">
2225		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2226		- </td>
2227		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2228		- <div class="mxchat-accordion-wrapper">
2229		- <div class="mxchat-content-preview">
2230		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2231		- <?php if (mb_strlen($content) > $preview_length) : ?>
2232		- <button class="mxchat-expand-toggle" type="button">
2233		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2234		- </button>
2235		- <?php endif; ?>
2236		- </div>
2237		- <div class="mxchat-content-full" style="display: none;">
2238		- <div class="content-view">
2239		- <?php
2240		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2241		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2242		- echo wp_kses_post(wpautop($content));
2243		- echo '</div>';
2244		- } else {
2245		- echo wp_kses_post(wpautop($content));
2246		- }
2247		- ?>
2248		- </div>
2249		- </div>
2250		- </div>
2251		- </td>
2252		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2253		- <?php
2254		- $actual_source = $source_url;
2255		- if (strpos($source_url, '_ungrouped_') === 0) {
2256		- $actual_source = $prompt->source_url ?? '';
2257		- }
2258		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
2259		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2260		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2261		- <?php esc_html_e('View', 'mxchat'); ?>
2262		- </a>
2263		- <?php else : ?>
2264		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
2265		- <?php endif; ?>
2266		- </td>
2267		- <td style="padding: 12px 16px;">
2268		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-ajax" data-vector-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_pinecone_prompt_nonce'); ?>" style="color: var(--mxch-error);">
2269		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2270		- </button>
2271		- </td>
2272		- </tr>
2273		- <?php
2274		- }
2275		- }
2276		- }
2277		- $html = ob_get_clean();
2278		-
2279		- // Generate pagination HTML for Pinecone
2280		- $total_pages = ceil($total_records / $per_page);
2281		- $pagination_html = '';
2282		- if ($total_pages > 1) {
2283		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
2284		-
2285		- // Previous button
2286		- if ($page > 1) {
2287		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
2288		- }
2289		-
2290		- // Page numbers
2291		- $start_page = max(1, $page - 2);
2292		- $end_page = min($total_pages, $page + 2);
2293		-
2294		- if ($start_page > 1) {
2295		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
2296		- if ($start_page > 2) {
2297		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2298		- }
2299		- }
2300		-
2301		- for ($i = $start_page; $i <= $end_page; $i++) {
2302		- if ($i == $page) {
2303		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
2304		- } else {
2305		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
2306		- }
2307		- }
2308		-
2309		- if ($end_page < $total_pages) {
2310		- if ($end_page < $total_pages - 1) {
2311		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2312		- }
2313		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
2314		- }
2315		-
2316		- // Next button
2317		- if ($page < $total_pages) {
2318		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
2319		- }
2320		-
2321		- $pagination_html .= '</div>';
2322		- }
2323		-
2324		- wp_send_json_success(array(
2325		- 'html' => $html,
2326		- 'pagination_html' => $pagination_html,
2327		- 'total_count' => $total_records,
2328		- 'total_pages' => $total_pages,
2329		- 'page' => $page,
2330		- 'per_page' => $per_page,
2331		- 'data_source' => 'pinecone'
2332		- ));
2333		-}
2334		-
2335		-/**
2336		- * AJAX handler for pagination - handles both WordPress DB and Pinecone data sources
2337		- * Returns paginated entries without requiring a full page reload
2338		- */
2339		-public function ajax_mxchat_paginate_entries() {
2340		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2341		-
2342		- if (!current_user_can('manage_options')) {
2343		- wp_send_json_error(array('message' => 'Unauthorized'));
2344		- return;
2345		- }
2346		-
2347		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2348		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2349		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2350		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2351		- $per_page = 25;
2352		-
2353		- // Check if Pinecone is enabled for this bot
2354		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
2355		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
2356		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2357		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
2358		-
2359		- if ($use_pinecone && $has_pinecone_api) {
2360		- // Delegate to Pinecone pagination handler (pass search params)
2361		- $_POST['page'] = $page;
2362		- $_POST['search'] = $search_query;
2363		- $_POST['content_type'] = $content_type_filter;
2364		- $this->ajax_mxchat_refresh_pinecone_entries();
2365		- return;
2366		- }
2367		-
2368		- // WordPress DB pagination - MUST match initial page load logic exactly
2369		- global $wpdb;
2370		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2371		- $offset = ($page - 1) * $per_page;
2372		-
2373		- // Build WHERE clause for search and content type filtering
2374		- $where_clauses = array();
2375		- $where_values = array();
2376		-
2377		- if ($search_query) {
2378		- $where_clauses[] = "article_content LIKE %s";
2379		- $where_values[] = '%' . $wpdb->esc_like($search_query) . '%';
2380		- }
2381		-
2382		- if ($content_type_filter) {
2383		- switch ($content_type_filter) {
2384		- case 'manual':
2385		- $where_clauses[] = "(source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')";
2386		- break;
2387		- case 'pdf':
2388		- $where_clauses[] = "source_url LIKE '%.pdf'";
2389		- break;
2390		- case 'url':
2391		- $where_clauses[] = "source_url != '' AND source_url IS NOT NULL AND source_url NOT LIKE 'mxchat://%' AND source_url NOT LIKE '%.pdf'";
2392		- break;
2393		- }
2394		- }
2395		-
2396		- $where_sql = !empty($where_clauses) ? 'WHERE ' . implode(' AND ', $where_clauses) : '';
2397		-
2398		- // Count grouped entries with filters applied
2399		- if (!empty($where_values)) {
2400		- $count_args = array_merge($where_values, $where_values);
2401		- $count_query = $wpdb->prepare(
2402		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2403		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))",
2404		- ...$count_args
2405		- );
2406		- $total_records = $wpdb->get_var($count_query);
2407		- } else if (!empty($where_sql)) {
2408		- // Content type filter only (no search), no prepared values needed
2409		- $total_records = $wpdb->get_var(
2410		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2411		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))"
2412		- );
2413		- } else {
2414		- // No filters
2415		- $total_records = $wpdb->get_var(
2416		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2417		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
2418		- );
2419		- }
2420		- $total_pages = ceil($total_records / $per_page);
2421		-
2422		- // Step 1: Get unique source_urls for this page (ordered by latest timestamp) with filters
2423		- if (!empty($where_values)) {
2424		- $query_args = array_merge($where_values, array($per_page, $offset));
2425		- $urls_query = $wpdb->prepare(
2426		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2427		- {$where_sql}
2428		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2429		- ...$query_args
2430		- );
2431		- } else if (!empty($where_sql)) {
2432		- $urls_query = $wpdb->prepare(
2433		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2434		- {$where_sql}
2435		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2436		- $per_page, $offset
2437		- );
2438		- } else {
2439		- $urls_query = $wpdb->prepare(
2440		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2441		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2442		- $per_page, $offset
2443		- );
2444		- }
2445		- $page_urls = $wpdb->get_results($urls_query);
2446		-
2447		- // Step 2: Build list of source_urls to fetch
2448		- $url_list = array();
2449		- $url_order_map = array();
2450		- $order_index = 0;
2451		- foreach ($page_urls as $url_row) {
2452		- $url = $url_row->source_url;
2453		- $url_list[] = $url;
2454		- $url_order_map[$url] = $order_index++;
2455		- }
2456		-
2457		- // Step 3: Fetch all rows for these source_urls (with search filter if applicable)
2458		- $prompts = array();
2459		- if (!empty($url_list)) {
2460		- $placeholders = implode(',', array_fill(0, count($url_list), '%s'));
2461		- if ($search_query) {
2462		- // Include search filter in the final fetch
2463		- $prompts_query = $wpdb->prepare(
2464		- "SELECT id, article_content, source_url, timestamp, role_restriction
2465		- FROM {$table_name}
2466		- WHERE source_url IN ($placeholders) AND article_content LIKE %s
2467		- ORDER BY timestamp DESC",
2468		- ...array_merge($url_list, ['%' . $wpdb->esc_like($search_query) . '%'])
2469		- );
2470		- } else {
2471		- $prompts_query = $wpdb->prepare(
2472		- "SELECT id, article_content, source_url, timestamp, role_restriction
2473		- FROM {$table_name}
2474		- WHERE source_url IN ($placeholders)
2475		- ORDER BY timestamp DESC",
2476		- $url_list
2477		- );
2478		- }
2479		- $prompts = $wpdb->get_results($prompts_query);
2480		- }
2481		-
2482		- // Group prompts by source_url for chunk display
2483		- $grouped_prompts = array();
2484		- foreach ($prompts as $prompt) {
2485		- $source_url = $prompt->source_url ?? '';
2486		-
2487		- // Parse chunk metadata using the proper chunker method (same as initial page load)
2488		- if (class_exists('MxChat_Chunker')) {
2489		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($prompt->article_content);
2490		- $prompt->chunk_metadata = $chunk_meta['metadata'];
2491		- $prompt->display_content = $chunk_meta['text'];
2492		- } else {
2493		- $prompt->chunk_metadata = array();
2494		- $prompt->display_content = $prompt->article_content;
2495		- }
2496		-
2497		- if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0) {
2498		- if (!isset($grouped_prompts[$source_url])) {
2499		- $grouped_prompts[$source_url] = array();
2500		- }
2501		- $grouped_prompts[$source_url][] = $prompt;
2502		- } else {
2503		- // Ungrouped entries
2504		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2505		- }
2506		- }
2507		-
2508		- // Sort groups by the original URL order (newest first)
2509		- uksort($grouped_prompts, function($a, $b) use ($url_order_map) {
2510		- $order_a = isset($url_order_map[$a]) ? $url_order_map[$a] : PHP_INT_MAX;
2511		- $order_b = isset($url_order_map[$b]) ? $url_order_map[$b] : PHP_INT_MAX;
2512		- return $order_a - $order_b;
2513		- });
2514		-
2515		- // Sort each group internally by chunk_index
2516		- foreach ($grouped_prompts as $source_url => &$group) {
2517		- usort($group, function($a, $b) {
2518		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2519		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2520		- return $index_a - $index_b;
2521		- });
2522		- }
2523		- unset($group);
2524		-
2525		- // Build HTML for the table rows
2526		- ob_start();
2527		- $display_index = 0;
2528		- $current_page = $page;
2529		- $data_source = 'wordpress';
2530		- $current_bot_id = $bot_id;
2531		- $preview_length = 150;
2532		-
2533		- if (empty($grouped_prompts)) {
2534		- echo '<tr><td colspan="5" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2535		- esc_html_e('No knowledge entries found. Use the Import Options to add content.', 'mxchat');
2536		- echo '</td></tr>';
2537		- } else {
2538		- foreach ($grouped_prompts as $source_url => $group) {
2539		- $chunk_count = count($group);
2540		- $first_prompt = $group[0];
2541		- $display_index++;
2542		-
2543		- if ($chunk_count > 1) {
2544		- // Multiple chunks - show grouped row with expand button
2545		- $group_id = 'group-' . md5($source_url);
2546		- ?>
2547		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2548		- class="mxchat-chunk-group-header"
2549		- data-source="<?php echo esc_attr($data_source); ?>"
2550		- data-group-id="<?php echo esc_attr($group_id); ?>"
2551		- style="border-bottom: 1px solid var(--mxch-card-border);">
2552		- <td style="padding: 12px 16px; text-align: center;">
2553		- <input type="checkbox"
2554		- class="mxchat-entry-checkbox"
2555		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2556		- data-source="<?php echo esc_attr($data_source); ?>"
2557		- data-source-url="<?php echo esc_attr($source_url); ?>"
2558		- data-is-group="true"
2559		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2560		- </td>
2561		- <td style="padding: 12px 16px; font-size: 13px;">
2562		- <?php echo esc_html($first_prompt->id); ?>
2563		- </td>
2564		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2565		- <div class="mxchat-chunk-group-info">
2566		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2567		- <span class="dashicons dashicons-arrow-right-alt2"></span>
2568		- </button>
2569		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2570		- <span class="mxchat-chunk-preview">
2571		- <?php
2572		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : $first_prompt->article_content;
2573		- $content_preview = mb_substr($parent_content, 0, 100);
2574		- echo esc_html($content_preview . '...');
2575		- ?>
2576		- </span>
2577		- </div>
2578		- </td>
2579		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2580		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2581		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2582		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2583		- <?php esc_html_e('View Source', 'mxchat'); ?>
2584		- </a>
2585		- <?php else : ?>
2586		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2587		- <?php endif; ?>
2588		- </td>
2589		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2590		- <?php if ($data_source !== 'pinecone') : ?>
2591		- <button type="button"
2592		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2593		- data-source-url="<?php echo esc_attr($source_url); ?>"
2594		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2595		- data-data-source="<?php echo esc_attr($data_source); ?>"
2596		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2597		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2598		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2599		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2600		- </button>
2601		- <?php endif; ?>
2602		- <button type="button"
2603		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2604		- data-source-url="<?php echo esc_attr($source_url); ?>"
2605		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2606		- data-data-source="<?php echo esc_attr($data_source); ?>"
2607		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2608		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2609		- style="color: var(--mxch-error);"
2610		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2611		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2612		- </button>
2613		- </td>
2614		- </tr>
2615		- <?php
2616		- // Render hidden chunk rows
2617		- foreach ($group as $chunk_index => $chunk) {
2618		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
2619		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
2620		- $content = isset($chunk->display_content) ? $chunk->display_content : $chunk->article_content;
2621		- $content_preview = mb_strlen($content) > $preview_length
2622		- ? mb_substr($content, 0, $preview_length) . '...'
2623		- : $content;
2624		- ?>
2625		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
2626		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
2627		- data-source="<?php echo esc_attr($data_source); ?>"
2628		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
2629		- <td style="padding: 12px 16px; text-align: center;">
2630		- <!-- Checkbox column placeholder for chunks (managed by group) -->
2631		- </td>
2632		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
2633		- <!-- Hidden ID column for chunks -->
2634		- </td>
2635		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2636		- <div class="mxchat-accordion-wrapper">
2637		- <div class="mxchat-content-preview">
2638		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
2639		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
2640		- </span>
2641		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2642		- <?php if (mb_strlen($content) > $preview_length) : ?>
2643		- <button class="mxchat-expand-toggle" type="button">
2644		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2645		- </button>
2646		- <?php endif; ?>
2647		- </div>
2648		- <div class="mxchat-content-full" style="display: none;">
2649		- <div class="content-view">
2650		- <?php
2651		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2652		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2653		- echo wp_kses_post(wpautop($content));
2654		- echo '</div>';
2655		- } else {
2656		- echo wp_kses_post(wpautop($content));
2657		- }
2658		- ?>
2659		- </div>
2660		- </div>
2661		- </div>
2662		- </td>
2663		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2664		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
2665		- </td>
2666		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
2667		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
2668		- </td>
2669		- </tr>
2670		- <?php
2671		- }
2672		- } else {
2673		- // Single entry - display normally with accordion
2674		- $prompt = $first_prompt;
2675		- $content = isset($prompt->display_content) ? $prompt->display_content : $prompt->article_content;
2676		- $content_preview = mb_strlen($content) > $preview_length
2677		- ? mb_substr($content, 0, $preview_length) . '...'
2678		- : $content;
2679		- ?>
2680		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
2681		- data-source="<?php echo esc_attr($data_source); ?>"
2682		- style="border-bottom: 1px solid var(--mxch-card-border);">
2683		- <td style="padding: 12px 16px; text-align: center;">
2684		- <input type="checkbox"
2685		- class="mxchat-entry-checkbox"
2686		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2687		- data-source="<?php echo esc_attr($data_source); ?>"
2688		- data-source-url="<?php echo esc_attr($source_url); ?>"
2689		- data-is-group="false">
2690		- </td>
2691		- <td style="padding: 12px 16px; font-size: 13px;">
2692		- <?php echo esc_html($prompt->id); ?>
2693		- </td>
2694		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2695		- <div class="mxchat-accordion-wrapper">
2696		- <div class="mxchat-content-preview">
2697		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2698		- <?php if (mb_strlen($content) > $preview_length) : ?>
2699		- <button class="mxchat-expand-toggle" type="button">
2700		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2701		- </button>
2702		- <?php endif; ?>
2703		- </div>
2704		- <div class="mxchat-content-full" style="display: none;">
2705		- <div class="content-view">
2706		- <?php
2707		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2708		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2709		- echo wp_kses_post(wpautop($content));
2710		- echo '</div>';
2711		- } else {
2712		- echo wp_kses_post(wpautop($content));
2713		- }
2714		- ?>
2715		- </div>
2716		- </div>
2717		- </div>
2718		- </td>
2719		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2720		- <?php
2721		- $actual_source = $source_url;
2722		- if (strpos($source_url, '_ungrouped_') === 0) {
2723		- $actual_source = $prompt->source_url ?? '';
2724		- }
2725		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
2726		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2727		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2728		- <?php esc_html_e('View', 'mxchat'); ?>
2729		- </a>
2730		- <?php else : ?>
2731		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
2732		- <?php endif; ?>
2733		- </td>
2734		- <td style="padding: 12px 16px; white-space: nowrap;">
2735		- <button type="button"
2736		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2737		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
2738		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2739		- data-data-source="<?php echo esc_attr($data_source); ?>"
2740		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2741		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2742		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2743		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2744		- </button>
2745		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-wordpress" data-entry-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_wordpress_prompt_nonce'); ?>" style="color: var(--mxch-error);">
2746		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2747		- </button>
2748		- </td>
2749		- </tr>
2750		- <?php
2751		- }
2752		- }
2753		- }
2754		- $html = ob_get_clean();
2755		-
2756		- // Generate pagination HTML (include search/filter data for subsequent pages)
2757		- $pagination_html = '';
2758		- if ($total_pages > 1) {
2759		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
2760		-
2761		- // Previous button
2762		- if ($page > 1) {
2763		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
2764		- }
2765		-
2766		- // Page numbers
2767		- $start_page = max(1, $page - 2);
2768		- $end_page = min($total_pages, $page + 2);
2769		-
2770		- if ($start_page > 1) {
2771		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
2772		- if ($start_page > 2) {
2773		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2774		- }
2775		- }
2776		-
2777		- for ($i = $start_page; $i <= $end_page; $i++) {
2778		- if ($i == $page) {
2779		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
2780		- } else {
2781		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
2782		- }
2783		- }
2784		-
2785		- if ($end_page < $total_pages) {
2786		- if ($end_page < $total_pages - 1) {
2787		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2788		- }
2789		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
2790		- }
2791		-
2792		- // Next button
2793		- if ($page < $total_pages) {
2794		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
2795		- }
2796		-
2797		- $pagination_html .= '</div>';
2798		- }
2799		-
2800		- wp_send_json_success(array(
2801		- 'html' => $html,
2802		- 'pagination_html' => $pagination_html,
2803		- 'total_count' => $total_records,
2804		- 'total_pages' => $total_pages,
2805		- 'page' => $page,
2806		- 'per_page' => $per_page,
2807		- 'data_source' => 'wordpress'
2808		- ));
2809		-}
2810		-
2811		-/**
2812		- * AJAX handler to detect available sitemaps on the site
2813		- * Optimized for speed - only checks primary sitemap indexes first
2814		- */
2815		-public function ajax_mxchat_detect_sitemaps() {
2816		- check_ajax_referer('mxchat_detect_sitemaps_nonce', 'nonce');
2817		-
2818		- if (!current_user_can('manage_options')) {
2819		- wp_send_json_error(array('message' => 'Unauthorized'));
2820		- return;
2821		- }
2822		-
2823		- $site_url = get_site_url();
2824		- $sitemaps = array();
2825		- $found_index = false;
2826		-
2827		- // Only check the main sitemap index files first (much faster)
2828		- // These are the primary entry points that contain sub-sitemaps
2829		- $primary_indexes = array(
2830		- 'sitemap_index.xml' => 'Yoast SEO / Rank Math', // Yoast & Rank Math
2831		- 'wp-sitemap.xml' => 'WordPress Core', // WordPress Core
2832		- 'sitemap.xml' => 'Standard', // Generic/AIOSEO
2833		- );
2834		-
2835		- foreach ($primary_indexes as $path => $source) {
2836		- $url = trailingslashit($site_url) . $path;
2837		-
2838		- $response = wp_remote_head($url, array(
2839		- 'timeout' => 10,
2840		- 'sslverify' => false,
2841		- 'redirection' => 1,
2842		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2843		- ));
2844		-
2845		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2846		- // Found a sitemap index - parse it to get sub-sitemaps
2847		- $sub_sitemaps = $this->parse_sitemap_index($url);
2848		- if (!empty($sub_sitemaps)) {
2849		- $sitemaps[] = array(
2850		- 'url' => $url,
2851		- 'type' => 'index',
2852		- 'source' => $source,
2853		- 'sub_sitemaps' => $sub_sitemaps
2854		- );
2855		- $found_index = true;
2856		- // Found a valid index, no need to check others
2857		- break;
2858		- }
2859		- }
2860		- }
2861		-
2862		- // If no sitemap index found, check for standalone sitemaps
2863		- if (!$found_index) {
2864		- $standalone_sitemaps = array(
2865		- 'post-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
2866		- 'page-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
2867		- );
2868		-
2869		- foreach ($standalone_sitemaps as $path => $info) {
2870		- $url = trailingslashit($site_url) . $path;
2871		-
2872		- $response = wp_remote_head($url, array(
2873		- 'timeout' => 2,
2874		- 'sslverify' => false
2875		- ));
2876		-
2877		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2878		- $sitemaps[] = array(
2879		- 'url' => $url,
2880		- 'type' => $info['type'],
2881		- 'source' => $info['source'],
2882		- 'url_count' => 0 // Skip URL count for speed
2883		- );
2884		- }
2885		- }
2886		- }
2887		-
2888		- wp_send_json_success(array(
2889		- 'sitemaps' => $sitemaps,
2890		- 'site_url' => $site_url
2891		- ));
2892		-}
2893		-
2894		-/**
2895		- * Parse a sitemap index to get sub-sitemaps
2896		- * Optimized: doesn't fetch URL count for each sub-sitemap (too slow)
2897		- */
2898		-private function parse_sitemap_index($url) {
2899		- $sub_sitemaps = array();
2900		-
2901		- $response = wp_remote_get($url, array(
2902		- 'timeout' => 30,
2903		- 'sslverify' => false,
2904		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2905		- 'headers' => array(
2906		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
2907		- 'Accept-Language' => 'en-US,en;q=0.9',
2908		- ),
2909		- ));
2910		-
2911		- if (is_wp_error($response)) {
2912		- return $sub_sitemaps;
2913		- }
2914		-
2915		- $body = wp_remote_retrieve_body($response);
2916		- if (empty($body)) {
2917		- return $sub_sitemaps;
2918		- }
2919		-
2920		- // Suppress XML errors
2921		- libxml_use_internal_errors(true);
2922		- $xml = simplexml_load_string($body);
2923		- libxml_clear_errors();
2924		-
2925		- if ($xml === false) {
2926		- return $sub_sitemaps;
2927		- }
2928		-
2929		- // Check if it's a sitemap index (contains <sitemap> elements)
2930		- if (isset($xml->sitemap)) {
2931		- foreach ($xml->sitemap as $sitemap) {
2932		- $loc = (string) $sitemap->loc;
2933		- if (!empty($loc)) {
2934		- // Try to determine the type from the URL
2935		- $type = 'content';
2936		- if (strpos($loc, 'category') !== false || strpos($loc, 'tag') !== false || strpos($loc, 'taxonomy') !== false) {
2937		- $type = 'taxonomy';
2938		- } elseif (strpos($loc, 'author') !== false || strpos($loc, 'user') !== false) {
2939		- $type = 'author';
2940		- }
2941		-
2942		- // Skip URL count - too slow to fetch for each sitemap
2943		- $sub_sitemaps[] = array(
2944		- 'url' => $loc,
2945		- 'type' => $type,
2946		- 'url_count' => 0, // Don't fetch - takes too long
2947		- 'name' => basename(parse_url($loc, PHP_URL_PATH))
2948		- );
2949		- }
2950		- }
2951		- }
2952		-
2953		- return $sub_sitemaps;
2954		-}
2955		-
2956		-/**
2957		- * Get URL count from a sitemap
2958		- */
2959		-private function get_sitemap_url_count($url) {
2960		- $response = wp_remote_get($url, array(
2961		- 'timeout' => 30,
2962		- 'sslverify' => false,
2963		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2964		- 'headers' => array(
2965		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
2966		- 'Accept-Language' => 'en-US,en;q=0.9',
2967		- ),
2968		- ));
2969		-
2970		- if (is_wp_error($response)) {
2971		- return 0;
2972		- }
2973		-
2974		- $body = wp_remote_retrieve_body($response);
2975		- if (empty($body)) {
2976		- return 0;
2977		- }
2978		-
2979		- // Count <url> or <loc> elements
2980		- $count = preg_match_all('/<url>/i', $body, $matches);
2981		- return $count ?: 0;
2982		-}
2983		-
2984		-/**
2985		- * Get sitemaps declared in robots.txt
2986		- */
2987		-private function get_sitemaps_from_robots($site_url) {
2988		- $sitemaps = array();
2989		- $robots_url = trailingslashit($site_url) . 'robots.txt';
2990		-
2991		- $response = wp_remote_get($robots_url, array(
2992		- 'timeout' => 15,
2993		- 'sslverify' => false,
2994		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2995		- ));
2996		-
2997		- if (is_wp_error($response)) {
2998		- return $sitemaps;
2999		- }
3000		-
3001		- $body = wp_remote_retrieve_body($response);
3002		- if (empty($body)) {
3003		- return $sitemaps;
3004		- }
3005		-
3006		- // Find Sitemap: declarations
3007		- if (preg_match_all('/^Sitemap:\s*(.+)$/mi', $body, $matches)) {
3008		- foreach ($matches[1] as $sitemap_url) {
3009		- $sitemap_url = trim($sitemap_url);
3010		- if (filter_var($sitemap_url, FILTER_VALIDATE_URL)) {
3011		- $sitemaps[] = $sitemap_url;
3012		- }
3013		- }
3014		- }
3015		-
3016		- return $sitemaps;
3017		-}
3018		-
3019	1767	public function mxchat_stop_processing() {
3020	1768	// Verify permissions
3021	1769	if (!current_user_can('manage_options')) {
3022	1770	wp_die(esc_html__('Unauthorized access', 'mxchat'));
		@@ -3024,52 +1772,28 @@
3024	1772
3025	1773	// Verify nonce
3026	1774	check_admin_referer('mxchat_stop_processing_action', 'mxchat_stop_processing_nonce');
3027	1775
3028		- global $wpdb;
3029		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
3030		-
3031		- // Get active queue IDs
3032		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
3033		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
3034		-
3035		- // Delete all pending items from active queues
3036		- if ($sitemap_queue_id) {
3037		- $wpdb->delete(
3038		- $table_name,
3039		- array(
3040		- 'queue_id' => $sitemap_queue_id,
3041		- 'status' => 'pending'
3042		- ),
3043		- array('%s', '%s')
3044		- );
3045		-
3046		- delete_transient('mxchat_active_queue_sitemap');
	1776	+ // Get the last sitemap URL and clear its transient
	1777	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
	1778	+ if ($sitemap_url) {
	1779	+ delete_transient('mxchat_sitemap_status_' . md5($sitemap_url));
3047	1780	delete_transient('mxchat_last_sitemap_url');
3048	1781	}
3049		-
3050		- if ($pdf_queue_id) {
3051		- // Get PDF path before deleting
3052		- $pdf_path = $this->mxchat_get_queue_meta($pdf_queue_id, 'pdf_path');
3053		-
3054		- $wpdb->delete(
3055		- $table_name,
3056		- array(
3057		- 'queue_id' => $pdf_queue_id,
3058		- 'status' => 'pending'
3059		- ),
3060		- array('%s', '%s')
3061		- );
3062		-
3063		- // Delete PDF file
3064		- if ($pdf_path && file_exists($pdf_path)) {
3065		- wp_delete_file($pdf_path);
3066		- }
3067		-
3068		- delete_transient('mxchat_active_queue_pdf');
	1782	+
	1783	+ // Get the last PDF URL and clear its transient
	1784	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1785	+ if ($pdf_url) {
	1786	+ delete_transient('mxchat_pdf_status_' . md5($pdf_url));
3069	1787	delete_transient('mxchat_last_pdf_url');
3070	1788	}
3071	1789
	1790	+ // Unschedule any pending sitemap events
	1791	+ $timestamp = wp_next_scheduled('mxchat_process_sitemap_urls');
	1792	+ if ($timestamp) {
	1793	+ wp_unschedule_event($timestamp, 'mxchat_process_sitemap_urls');
	1794	+ }
	1795	+
3072	1796	// Redirect back with a success message
3073	1797	set_transient('mxchat_admin_notice_success',
3074	1798	esc_html__('Processing has been stopped successfully.', 'mxchat'),
3075	1799	30
		@@ -3076,12 +1800,8 @@
3076	1800	);
3077	1801	wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
3078	1802	exit;
3079	1803	}
3080		-
3081		-/**
3082		- * Get content list for processing
3083		- */
3084	1804	public function ajax_mxchat_get_content_list() {
3085	1805	// Verify the nonce
3086	1806	check_ajax_referer('mxchat_content_selector_nonce', 'nonce');
3087	1807
		@@ -3089,9 +1809,9 @@
3089	1809	wp_send_json_error(__('Unauthorized access', 'mxchat'));
3090	1810	}
3091	1811
3092	1812	$page = isset($_GET['page']) ? absint($_GET['page']) : 1;
3093		- $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 100;
	1813	+ $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 20;
3094	1814	$search = isset($_GET['search']) ? sanitize_text_field($_GET['search']) : '';
3095	1815	$post_type = isset($_GET['post_type']) ? sanitize_text_field($_GET['post_type']) : 'all';
3096	1816	$post_status = isset($_GET['post_status']) ? sanitize_text_field($_GET['post_status']) : 'publish';
3097	1817	$processed_filter = isset($_GET['processed_filter']) ? sanitize_text_field($_GET['processed_filter']) : 'all';
		@@ -3104,47 +1824,20 @@
3104	1824	'orderby' => 'date',
3105	1825	'order' => 'DESC',
3106	1826	);
3107	1827
3108		- // Handle post types - IMPROVED VERSION
	1828	+ // Handle post types
3109	1829	if ($post_type !== 'all') {
3110	1830	$args['post_type'] = $post_type;
3111	1831	} else {
3112		- // Get all available post types that might contain content
3113		- $all_post_types = array();
	1832	+ // Default to post and page if we can't get post types
	1833	+ $args['post_type'] = array('post', 'page');
3114	1834
3115		- // First get all public post types
3116		- $public_types = get_post_types(array('public' => true), 'names');
3117		- $all_post_types = array_merge($all_post_types, $public_types);
3118		-
3119		- // Add common forum/community post types
3120		- $forum_types = array('topic', 'reply', 'forum', 'wpforo_topic', 'wpforo_post');
3121		- foreach ($forum_types as $forum_type) {
3122		- if (post_type_exists($forum_type)) {
3123		- $all_post_types[] = $forum_type;
3124		- }
	1835	+ // Try to get public post types
	1836	+ $public_types = $this->mxchat_get_public_post_types();
	1837	+ if (is_array($public_types) && !empty($public_types)) {
	1838	+ $args['post_type'] = array_keys($public_types);
3125	1839	}
3126		-
3127		- // Add other commonly used post types
3128		- $common_types = array('product', 'job_listing', 'event', 'portfolio');
3129		- foreach ($common_types as $common_type) {
3130		- if (post_type_exists($common_type)) {
3131		- $all_post_types[] = $common_type;
3132		- }
3133		- }
3134		-
3135		- // Remove duplicates and ensure we have at least some post types
3136		- $all_post_types = array_unique($all_post_types);
3137		-
3138		- if (empty($all_post_types)) {
3139		- // Fallback to basic post types
3140		- $all_post_types = array('post', 'page');
3141		- }
3142		-
3143		- $args['post_type'] = $all_post_types;
3144		-
3145		- // Debug logging to see what post types are being queried
3146		- //error_log('MxChat Debug: Querying post types: ' . implode(', ', $all_post_types));
3147	1840	}
3148	1841
3149	1842	if (!empty($search)) {
3150	1843	$args['s'] = $search;
		@@ -3149,58 +1842,36 @@
3149	1842	if (!empty($search)) {
3150	1843	$args['s'] = $search;
3151	1844	}
3152	1845
3153		- // Get processed data from storage
	1846	+ // ================================
	1847	+ // FIXED: Check only the ACTIVE storage method
	1848	+ // ================================
	1849	+
3154	1850	$processed_data = array();
3155	1851
	1852	+ // Check if Pinecone is enabled
3156	1853	$pinecone_options = get_option('mxchat_pinecone_addon_options', array());
3157	1854	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3158		-
	1855	+
3159	1856	if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3160		- // Get fresh data from Pinecone - no caching
	1857	+ // ONLY check Pinecone if it's enabled
3161	1858	$processed_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
3162	1859	} else {
3163		- // WordPress DB checking with better URL matching for all post types
	1860	+ // ONLY check WordPress DB if Pinecone is not enabled
3164	1861	global $wpdb;
3165	1862	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3166		- $processed_items = $wpdb->get_results("SELECT id, source_url, article_content, timestamp FROM {$table_name}");
3167		-
3168		- // Group items by source_url to count chunks
3169		- $url_chunk_counts = array();
3170		- $url_latest_timestamp = array();
3171		- $url_first_id = array();
3172		-
	1863	+ $processed_items = $wpdb->get_results("SELECT id, source_url, timestamp FROM {$table_name}");
	1864	+
3173	1865	if (!empty($processed_items)) {
3174	1866	foreach ($processed_items as $item) {
3175		- $url = $item->source_url;
3176		- if (empty($url)) continue;
3177		-
3178		- // Count chunks per URL
3179		- if (!isset($url_chunk_counts[$url])) {
3180		- $url_chunk_counts[$url] = 0;
3181		- $url_latest_timestamp[$url] = $item->timestamp;
3182		- $url_first_id[$url] = $item->id;
3183		- }
3184		- $url_chunk_counts[$url]++;
3185		-
3186		- // Track latest timestamp
3187		- if (strtotime($item->timestamp) > strtotime($url_latest_timestamp[$url])) {
3188		- $url_latest_timestamp[$url] = $item->timestamp;
3189		- }
3190		- }
3191		-
3192		- // Now build processed_data with chunk counts
3193		- foreach ($url_chunk_counts as $url => $chunk_count) {
3194		- $post_id = $this->mxchat_url_to_post_id_improved($url);
3195		-
	1867	+ $post_id = url_to_postid($item->source_url);
3196	1868	if ($post_id) {
3197	1869	$processed_data[$post_id] = array(
3198		- 'db_id' => $url_first_id[$url],
3199		- 'timestamp' => $url_latest_timestamp[$url],
3200		- 'url' => $url,
3201		- 'source' => 'wordpress',
3202		- 'chunk_count' => $chunk_count
	1870	+ 'db_id' => $item->id,
	1871	+ 'timestamp' => $item->timestamp,
	1872	+ 'url' => $item->source_url,
	1873	+ 'source' => 'wordpress'
3203	1874	);
3204	1875	}
3205	1876	}
3206	1877	}
		@@ -3205,8 +1876,10 @@
3205	1876	}
3206	1877	}
3207	1878	}
3208	1879
	1880	+ // ================================
	1881	+
3209	1882	// Get processed IDs as a simple array for in_array checks
3210	1883	$processed_ids = array_keys($processed_data);
3211	1884
3212	1885	// Handle processed/unprocessed filter
		@@ -3248,14 +1921,8 @@
3248	1921	$db_record_id = $item_data['db_id'];
3249	1922	}
3250	1923	}
3251	1924
3252		- // Get chunk count for this item
3253		- $chunk_count = 0;
3254		- if ($is_processed && isset($processed_data[$id]['chunk_count'])) {
3255		- $chunk_count = intval($processed_data[$id]['chunk_count']);
3256		- }
3257		-
3258	1925	$content_items[] = array(
3259	1926	'id' => $id,
3260	1927	'title' => get_the_title(),
3261	1928	'permalink' => get_permalink(),
		@@ -3266,10 +1933,9 @@
3266	1933	'word_count' => $word_count,
3267	1934	'already_processed' => $is_processed,
3268	1935	'processed_date' => $processed_date,
3269	1936	'db_record_id' => $db_record_id,
3270		- 'data_source' => $data_source,
3271		- 'chunk_count' => $chunk_count
	1937	+ 'data_source' => $data_source
3272	1938	);
3273	1939	}
3274	1940	wp_reset_postdata();
3275	1941	}
		@@ -3285,178 +1951,8 @@
3285	1951	wp_send_json_success($response);
3286	1952	exit;
3287	1953	}
3288	1954
3289		-
3290		-/**
3291		- * This function handles various WooCommerce URL formats and permalink structures
3292		- */
3293		-private function mxchat_url_to_post_id_improved($url) {
3294		- // First try the standard WordPress function
3295		- $post_id = url_to_postid($url);
3296		-
3297		- if ($post_id > 0) {
3298		- return $post_id;
3299		- }
3300		-
3301		- // If that fails, try more aggressive URL matching
3302		- // Remove trailing slashes and query parameters for better matching
3303		- $clean_url = rtrim($url, '/');
3304		- $clean_url = strtok($clean_url, '?'); // Remove query parameters
3305		-
3306		- // Try again with cleaned URL
3307		- $post_id = url_to_postid($clean_url);
3308		- if ($post_id > 0) {
3309		- return $post_id;
3310		- }
3311		-
3312		- // For bbPress forum topics, try extracting slug from URL
3313		- if (strpos($url, '/topic/') !== false || strpos($url, '/forums/') !== false) {
3314		- // Handle bbPress URLs: /forums/topic/topic-name/
3315		- if (preg_match('/\/forums\/topic\/([^\/\?]+)/', $url, $matches)) {
3316		- $topic_slug = $matches[1];
3317		-
3318		- // Look up topic by slug
3319		- $topic = get_page_by_path($topic_slug, OBJECT, 'topic');
3320		- if ($topic) {
3321		- return $topic->ID;
3322		- }
3323		-
3324		- // Alternative method: query by post_name
3325		- global $wpdb;
3326		- $post_id = $wpdb->get_var($wpdb->prepare(
3327		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3328		- $topic_slug
3329		- ));
3330		-
3331		- if ($post_id) {
3332		- return intval($post_id);
3333		- }
3334		- }
3335		-
3336		- // Handle simpler topic URLs: /topic/topic-name/
3337		- if (preg_match('/\/topic\/([^\/\?]+)/', $url, $matches)) {
3338		- $topic_slug = $matches[1];
3339		-
3340		- global $wpdb;
3341		- $post_id = $wpdb->get_var($wpdb->prepare(
3342		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3343		- $topic_slug
3344		- ));
3345		-
3346		- if ($post_id) {
3347		- return intval($post_id);
3348		- }
3349		- }
3350		- }
3351		-
3352		- // For WooCommerce products
3353		- if (strpos($url, '/product/') !== false || strpos($url, 'product=') !== false) {
3354		- // Extract product slug from various URL formats
3355		- $product_slug = '';
3356		-
3357		- // Handle pretty permalinks: /product/product-name/
3358		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
3359		- $product_slug = $matches[1];
3360		- }
3361		- // Handle query parameters: ?product=product-name
3362		- elseif (preg_match('/[\?&]product=([^&]+)/', $url, $matches)) {
3363		- $product_slug = $matches[1];
3364		- }
3365		-
3366		- if (!empty($product_slug)) {
3367		- // Look up product by slug
3368		- $product = get_page_by_path($product_slug, OBJECT, 'product');
3369		- if ($product) {
3370		- return $product->ID;
3371		- }
3372		-
3373		- // Alternative method: query by post_name
3374		- global $wpdb;
3375		- $post_id = $wpdb->get_var($wpdb->prepare(
3376		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'product' AND post_status = 'publish'",
3377		- $product_slug
3378		- ));
3379		-
3380		- if ($post_id) {
3381		- return intval($post_id);
3382		- }
3383		- }
3384		- }
3385		-
3386		- // Generic approach: try to extract slug and match against all post types
3387		- $parsed_url = wp_parse_url($clean_url);
3388		- $path = $parsed_url['path'] ?? '';
3389		-
3390		- if (!empty($path)) {
3391		- // Get the last part of the path as potential slug
3392		- $path_parts = array_filter(explode('/', trim($path, '/')));
3393		- $potential_slug = end($path_parts);
3394		-
3395		- if (!empty($potential_slug)) {
3396		- global $wpdb;
3397		-
3398		- // Try to find any post with this slug
3399		- $post_id = $wpdb->get_var($wpdb->prepare(
3400		- "SELECT ID FROM {$wpdb->posts}
3401		- WHERE post_name = %s
3402		- AND post_status IN ('publish', 'closed', 'private')
3403		- AND post_type NOT IN ('revision', 'attachment', 'nav_menu_item')
3404		- ORDER BY CASE
3405		- WHEN post_type = 'post' THEN 1
3406		- WHEN post_type = 'page' THEN 2
3407		- WHEN post_type = 'topic' THEN 3
3408		- WHEN post_type = 'product' THEN 4
3409		- ELSE 5
3410		- END
3411		- LIMIT 1",
3412		- $potential_slug
3413		- ));
3414		-
3415		- if ($post_id) {
3416		- return intval($post_id);
3417		- }
3418		- }
3419		- }
3420		-
3421		- // ADDITIONAL: Try direct database lookup by URL variations
3422		- global $wpdb;
3423		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3424		-
3425		- // Try variations of the URL (with/without trailing slash, http/https)
3426		- $url_variations = array(
3427		- $url,
3428		- rtrim($url, '/'),
3429		- $url . '/',
3430		- str_replace('http://', 'https://', $url),
3431		- str_replace('https://', 'http://', $url),
3432		- str_replace('http://', 'https://', rtrim($url, '/')),
3433		- str_replace('https://', 'http://', rtrim($url, '/'))
3434		- );
3435		-
3436		- // Remove duplicates
3437		- $url_variations = array_unique($url_variations);
3438		-
3439		- foreach ($url_variations as $variation) {
3440		- $existing_record = $wpdb->get_row($wpdb->prepare(
3441		- "SELECT id, source_url FROM $table_name WHERE source_url = %s",
3442		- $variation
3443		- ));
3444		-
3445		- if ($existing_record) {
3446		- // Try to get post ID from this stored URL
3447		- $stored_post_id = url_to_postid($existing_record->source_url);
3448		- if ($stored_post_id > 0) {
3449		- return $stored_post_id;
3450		- }
3451		- }
3452		- }
3453		-
3454		- return 0; // No match found
3455		-}
3456		-/**
3457		- * Process selected content via AJAX
3458		- */
3459	1955	public function ajax_mxchat_process_selected_content() {
3460	1956	// Basic request validation
3461	1957	if (!check_ajax_referer('mxchat_content_selector_nonce', 'nonce', false)) {
3462	1958	wp_send_json_error('Invalid nonce');
		@@ -3480,11 +1976,8 @@
3480	1976	wp_send_json_error('No content selected');
3481	1977	exit;
3482	1978	}
3483	1979
3484		- // Get bot_id from request
3485		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
3486		-
3487	1980	// Process only ONE post at a time to avoid request size issues
3488	1981	$post_id = reset($post_ids);
3489	1982	$post = get_post($post_id);
3490	1983
		@@ -3491,151 +1984,15 @@
3491	1984	if (!$post) {
3492	1985	wp_send_json_error('Post not found');
3493	1986	exit;
3494	1987	}
3495		-
3496		- // Allow developers to modify post data before processing into knowledge base
3497		- $post = apply_filters('mxchat_before_process_post', $post, $bot_id);
3498		-
3499		- // Get content including title, short description (for WooCommerce), and main content
3500		- $content = $post->post_title . "\n\n";
3501		-
3502		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
3503		- if (!empty($post->post_excerpt)) {
3504		- // Remove shortcode tags but preserve content inside them
3505		- $clean_excerpt = $this->strip_shortcode_tags_preserve_content($post->post_excerpt);
3506		- $content .= "Short Description: " . wp_strip_all_tags($clean_excerpt) . "\n\n";
3507		- }
3508		-
3509		- // Add main content - remove shortcode tags but preserve content inside them
3510		- $clean_content = $this->strip_shortcode_tags_preserve_content($post->post_content);
3511		- $content .= wp_strip_all_tags($clean_content);
3512		-
3513		- // ADD WOOCOMMERCE PRODUCT DATA (pricing, stock, categories, custom tabs)
3514		- if (get_post_type($post_id) === 'product' && class_exists('WooCommerce')) {
3515		- $product = wc_get_product($post_id);
3516		-
3517		- if ($product) {
3518		- // Get pricing information
3519		- $regular_price = $product->get_regular_price();
3520		- $sale_price = $product->get_sale_price();
3521		- $price = $product->get_price();
3522		- $sku = $product->get_sku();
3523		-
3524		- // Get currency symbol
3525		- $currency_symbol = get_woocommerce_currency_symbol();
3526		-
3527		- // Add pricing information
3528		- $content .= "\n";
3529		- if (!empty($regular_price)) {
3530		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
3531		- } elseif (!empty($price)) {
3532		- $content .= "Price: " . $currency_symbol . $price . "\n";
3533		- }
3534		-
3535		- if (!empty($sale_price) && $sale_price !== $regular_price) {
3536		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
3537		- }
3538		-
3539		- // Handle variable products - show price range
3540		- if ($product->is_type('variable')) {
3541		- $min_price = $product->get_variation_price('min');
3542		- $max_price = $product->get_variation_price('max');
3543		- if ($min_price !== $max_price) {
3544		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
3545		- }
3546		- }
3547		-
3548		- if (!empty($sku)) {
3549		- $content .= "SKU: " . $sku . "\n";
3550		- }
3551		-
3552		- // Get product categories
3553		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
3554		- if (!empty($categories) && !is_wp_error($categories)) {
3555		- $content .= "Categories: " . implode(', ', $categories) . "\n";
3556		- }
3557		- }
3558		-
3559		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
3560		- $custom_tabs = get_post_meta($post_id, 'yikes_woo_products_tabs', true);
3561		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
3562		- foreach ($custom_tabs as $tab) {
3563		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
3564		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
3565		-
3566		- if (!empty($tab_title) && !empty($tab_content)) {
3567		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
3568		- }
3569		- }
3570		- }
3571		-
3572		- // Also check for reusable/saved tabs applied to this product
3573		- $applied_saved_tabs = get_post_meta($post_id, 'yikes_woo_reusable_products_tabs_applied', true);
3574		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
3575		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
3576		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
3577		- foreach ($applied_saved_tabs as $saved_tab_id) {
3578		- if (isset($saved_tabs[$saved_tab_id])) {
3579		- $tab = $saved_tabs[$saved_tab_id];
3580		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
3581		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
3582		-
3583		- if (!empty($tab_title) && !empty($tab_content)) {
3584		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
3585		- }
3586		- }
3587		- }
3588		- }
3589		- }
3590		- }
3591		-
3592		- // ADD ACF FIELDS SUPPORT
3593		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
3594		- if (!empty($acf_fields)) {
3595		- $acf_content_parts = array();
3596		-
3597		- foreach ($acf_fields as $field_name => $field_value) {
3598		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
3599		-
3600		- if (!empty($formatted_value)) {
3601		- $field_label = ucwords(str_replace('_', ' ', $field_name));
3602		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
3603		- }
3604		- }
3605		-
3606		- if (!empty($acf_content_parts)) {
3607		- $content .= "\n\n" . implode("\n", $acf_content_parts);
3608		- }
3609		- }
3610		-
3611		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
3612		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
3613		- if (!empty($custom_meta)) {
3614		- $meta_content_parts = array();
3615		-
3616		- foreach ($custom_meta as $meta_key => $meta_value) {
3617		- // Convert meta key to readable label
3618		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
3619		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
3620		- }
3621		-
3622		- if (!empty($meta_content_parts)) {
3623		- $content .= "\n\n" . implode("\n", $meta_content_parts);
3624		- }
3625		- }
3626		-
3627		- // Debug logging for WordPress Import content
3628		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Post ID: ' . $post_id . ' Title: ' . $post->post_title);
3629		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Raw post_content length: ' . strlen($post->post_content));
3630		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Final content length: ' . strlen($content));
3631		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Content preview: ' . substr($content, 0, 300));
3632		-
3633		- // Note: Removed 10,000 char limit - chunking now handles large content properly
3634		-
3635		- // Get bot-specific API key
3636		- $bot_options = $this->get_bot_options($bot_id);
3637		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	1988	+
	1989	+ // Get minimal content
	1990	+ $content = $post->post_title . "\n\n" . wp_strip_all_tags($post->post_content);
	1991	+ $content = substr($content, 0, 10000); // Limit content size
	1992	+
	1993	+ // Get API key with proper model detection
	1994	+ $options = get_option('mxchat_options');
3638	1995	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
3639	1996
3640	1997	if (strpos($selected_model, 'voyage') === 0) {
3641	1998	$api_key = $options['voyage_api_key'] ?? '';
		@@ -3648,9 +2005,8 @@
3648	2005	$provider_name = 'OpenAI';
3649	2006	}
3650	2007
3651	2008	if (empty($api_key)) {
3652		- MxChat_Admin::mxchat_log_debug('api_error', $provider_name . ' API key not configured for knowledge processing');
3653	2009	wp_send_json_error($provider_name . ' API key not configured');
3654	2010	exit;
3655	2011	}
3656	2012
		@@ -3656,23 +2012,26 @@
3656	2012
3657	2013	$source_url = get_permalink($post_id);
3658	2014	$vector_id = md5($source_url); // Vector ID for Pinecone
3659	2015
3660		- // Check for existing content in bot-specific storage
	2016	+ // ================================
	2017	+ // FIXED: Check for existing content in ONLY the active storage method
	2018	+ // ================================
	2019	+
3661	2020	$is_update = false;
3662	2021
3663		- // Get bot-specific Pinecone configuration
3664		- $bot_pinecone_config = $this->get_bot_pinecone_config($bot_id);
3665		- $use_pinecone = !empty($bot_pinecone_config) && ($bot_pinecone_config['use_pinecone'] ?? false);
	2022	+ // Check if Pinecone is enabled
	2023	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	2024	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3666	2025
3667		- if ($use_pinecone && !empty($bot_pinecone_config['api_key'])) {
3668		- // Check Pinecone for this bot
3669		- $pinecone_data = $this->mxchat_get_pinecone_processed_content($bot_pinecone_config);
	2026	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2027	+ // ONLY check Pinecone if it's enabled
	2028	+ $pinecone_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
3670	2029	if (isset($pinecone_data[$post_id])) {
3671	2030	$is_update = true;
3672	2031	}
3673	2032	} else {
3674		- // Check WordPress DB (same as before since it's shared)
	2033	+ // ONLY check WordPress DB if Pinecone is not enabled
3675	2034	global $wpdb;
3676	2035	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3677	2036	$existing_record = $wpdb->get_row($wpdb->prepare(
3678	2037	"SELECT id FROM $table_name WHERE source_url = %s",
		@@ -3682,54 +2041,54 @@
3682	2041	if ($existing_record) {
3683	2042	$is_update = true;
3684	2043	}
3685	2044	}
3686		-
3687		- // UPDATED 2.5.6: Determine content type based on post_type
3688		- $post_type = $post->post_type;
3689		- $content_type = 'content'; // Default fallback
3690		-
3691		- // Map WordPress post types to content types
3692		- switch ($post_type) {
3693		- case 'post':
3694		- $content_type = 'post';
3695		- break;
3696		- case 'page':
3697		- $content_type = 'page';
3698		- break;
3699		- case 'product':
3700		- $content_type = 'product';
3701		- break;
3702		- default:
3703		- // For custom post types, use the post type name
3704		- $content_type = sanitize_key($post_type);
3705		- break;
3706		- }
3707		-
3708		- // Use the centralized utility function with bot_id and content_type
	2045	+
	2046	+ // Use the centralized utility function for storage
3709	2047	$result = MxChat_Utils::submit_content_to_db(
3710		- $content,
3711		- $source_url,
	2048	+ $content,
	2049	+ $source_url,
3712	2050	$api_key,
3713		- $vector_id,
3714		- $bot_id,
3715		- $content_type
	2051	+ $vector_id
3716	2052	);
3717		-
	2053	+
3718	2054	if (is_wp_error($result)) {
3719		- MxChat_Admin::mxchat_log_debug('storage_error', 'Knowledge storage failed: ' . $result->get_error_message(), array('source_url' => $source_url));
3720	2055	wp_send_json_error('Storage failed: ' . $result->get_error_message());
3721	2056	exit;
3722	2057	}
3723		-
3724		- // Automatically apply role restriction based on tags
3725		- $this->apply_role_restriction_to_post($post_id, $source_url);
3726		-
	2058	+
	2059	+ // ================================
	2060	+ // UPDATE: Only update caches if Pinecone is enabled
	2061	+ // ================================
	2062	+
	2063	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2064	+ // Update vector ID cache for improved fetching
	2065	+ $this->mxchat_update_pinecone_vector_cache($vector_id);
	2066	+
	2067	+ // Update local processed content cache for immediate UI feedback
	2068	+ $pinecone_cache = get_option('mxchat_pinecone_processed_cache', array());
	2069	+ $pinecone_cache[$post_id] = array(
	2070	+ 'db_id' => $vector_id,
	2071	+ 'processed_date' => 'Just now',
	2072	+ 'url' => $source_url,
	2073	+ 'source' => 'pinecone',
	2074	+ 'timestamp' => current_time('timestamp')
	2075	+ );
	2076	+ update_option('mxchat_pinecone_processed_cache', $pinecone_cache);
	2077	+
	2078	+ // Also update the general processed content cache
	2079	+ $processed_cache = get_option('mxchat_processed_content_cache', array());
	2080	+ $processed_cache[$post_id] = array(
	2081	+ 'db_id' => $vector_id,
	2082	+ 'timestamp' => current_time('timestamp'),
	2083	+ 'url' => $source_url,
	2084	+ 'source' => 'pinecone'
	2085	+ );
	2086	+ update_option('mxchat_processed_content_cache', $processed_cache);
	2087	+ }
	2088	+
3727	2089	$operation_type = $is_update ? 'update' : 'new';
3728	2090
3729		- // Count ACF fields for debugging
3730		- $acf_field_count = count($acf_fields);
3731		-
3732	2091	// Success response with minimal data
3733	2092	wp_send_json_success(array(
3734	2093	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
3735	2094	'post_id' => $post_id,
		@@ -3734,94 +2093,27 @@
3734	2093	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
3735	2094	'post_id' => $post_id,
3736	2095	'title' => $post->post_title,
3737	2096	'operation_type' => $operation_type,
3738		- 'vector_id' => $vector_id,
3739		- 'acf_fields_found' => $acf_field_count,
3740		- 'content_preview' => substr($content, 0, 100) . '...',
3741		- 'bot_id' => $bot_id
	2097	+ 'vector_id' => $vector_id, // Include vector ID for debugging
	2098	+ 'cache_updated' => $use_pinecone // Indicate if cache was updated
3742	2099	));
3743	2100	exit;
3744	2101	}
3745	2102
3746		-private function apply_role_restriction_to_post($post_id, $source_url) {
3747		- // Get tag-role mappings
3748		- $mappings = get_option('mxchat_tag_role_mappings', array());
3749		-
3750		- if (empty($mappings)) {
3751		- return; // No mappings, leave as public
3752		- }
3753		-
3754		- // Get all tags for the post
3755		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
3756		-
3757		- if (empty($post_tags)) {
3758		- return; // No tags, leave as public
3759		- }
3760		-
3761		- // Determine the highest role restriction based on tags
3762		- $highest_role = 'public';
3763		- $role_hierarchy = array(
3764		- 'public' => 0,
3765		- 'logged_in' => 1,
3766		- 'subscriber' => 2,
3767		- 'contributor' => 3,
3768		- 'author' => 4,
3769		- 'editor' => 5,
3770		- 'administrator' => 6
3771		- );
3772		-
3773		- foreach ($post_tags as $tag_slug) {
3774		- if (isset($mappings[$tag_slug])) {
3775		- $role = $mappings[$tag_slug];
3776		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
3777		- $highest_role = $role;
3778		- }
3779		- }
3780		- }
3781		-
3782		- // If no restricted tags found, return (leave as public)
3783		- if ($highest_role === 'public') {
3784		- return;
3785		- }
3786		-
3787		- // Update the role restriction in the database
3788		- global $wpdb;
3789		-
3790		- // Check if using Pinecone
3791		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
3792		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3793		-
3794		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3795		- // Update Pinecone role restriction
3796		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
3797		- $vector_id = md5($source_url);
3798		-
3799		- $wpdb->replace(
3800		- $roles_table,
3801		- array(
3802		- 'vector_id' => $vector_id,
3803		- 'role_restriction' => $highest_role,
3804		- 'updated_at' => current_time('mysql')
3805		- ),
3806		- array('%s', '%s', '%s')
3807		- );
3808		- } else {
3809		- // Update WordPress DB
3810		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3811		-
3812		- $wpdb->update(
3813		- $table_name,
3814		- array('role_restriction' => $highest_role),
3815		- array('source_url' => $source_url),
3816		- array('%s'),
3817		- array('%s')
3818		- );
3819		- }
3820		-}
3821	2103
	2104	+
	2105	+ /**
	2106	+ * Updates cache with new vector ID if absent
	2107	+ */
	2108	+ public function mxchat_update_pinecone_vector_cache($vector_id) {
	2109	+ $cached_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2110	+ if (!in_array($vector_id, $cached_ids)) {
	2111	+ $cached_ids[] = $vector_id;
	2112	+ update_option('mxchat_pinecone_vector_ids_cache', $cached_ids);
	2113	+ }
	2114	+ }
3822	2115	public function mxchat_get_public_post_types() {
3823		- // Get all public post types
3824	2116	$post_types = get_post_types(array('public' => true), 'objects');
3825	2117	$post_type_options = array();
3826	2118
3827	2119	foreach ($post_types as $post_type) {
		@@ -3827,47 +2119,41 @@
3827	2119	foreach ($post_types as $post_type) {
3828	2120	$post_type_options[$post_type->name] = $post_type->label;
3829	2121	}
3830	2122
3831		- // Also include common forum/community post types that might not be marked as public
3832		- $additional_types = array(
3833		- 'topic' => 'Forum Topics (bbPress)',
3834		- 'reply' => 'Forum Replies (bbPress)',
3835		- 'forum' => 'Forums (bbPress)',
3836		- 'wpforo_topic' => 'wpForo Topics',
3837		- 'wpforo_post' => 'wpForo Posts'
3838		- );
3839		-
3840		- foreach ($additional_types as $type_name => $type_label) {
3841		- if (post_type_exists($type_name) && !isset($post_type_options[$type_name])) {
3842		- $post_type_options[$type_name] = $type_label;
3843		- }
3844		- }
3845		-
3846	2123	return $post_type_options;
3847	2124	}
3848		-
3849		-/**
3850		- * Retrieves processed content from Pinecone API
3851		- */
3852	2125	public function mxchat_get_pinecone_processed_content($pinecone_options) {
	2126	+ // First check local cache for immediate updates
	2127	+ $cached_data = get_option('mxchat_pinecone_processed_cache', array());
	2128	+
3853	2129	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
3854	2130	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
3855		-
	2131	+
3856	2132	if (empty($api_key) || empty($host)) {
3857		- return array();
	2133	+ // Return only cached data if API credentials are missing
	2134	+ return $cached_data;
3858	2135	}
3859		-
	2136	+
3860	2137	$pinecone_data = array();
3861		-
	2138	+
3862	2139	try {
3863		- // Always get fresh data from Pinecone
3864		- $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
3865		-
3866		- // Method 2: Final fallback - try stats endpoint (if available)
	2140	+ // Method 1: Try to get vectors using cached vector IDs first
	2141	+ $cached_vector_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2142	+
	2143	+ if (!empty($cached_vector_ids)) {
	2144	+ $pinecone_data = $this->mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $cached_vector_ids);
	2145	+ }
	2146	+
	2147	+ // Method 2: If no cached IDs or fetch failed, use scanning approach
3867	2148	if (empty($pinecone_data)) {
	2149	+ $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
	2150	+ }
	2151	+
	2152	+ // Method 3: Final fallback - try stats endpoint (if available)
	2153	+ if (empty($pinecone_data)) {
3868	2154	$stats_url = "https://{$host}/describe_index_stats";
3869		-
	2155	+
3870	2156	$response = wp_remote_post($stats_url, array(
3871	2157	'headers' => array(
3872	2158	'Api-Key' => $api_key,
3873	2159	'Content-Type' => 'application/json'
		@@ -3874,42 +2160,59 @@
3874	2160	),
3875	2161	'body' => json_encode(array()),
3876	2162	'timeout' => 30
3877	2163	));
3878		-
	2164	+
3879	2165	if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3880	2166	$body = wp_remote_retrieve_body($response);
3881	2167	$stats_data = json_decode($body, true);
	2168	+
	2169	+ // Log stats for debugging but don't rely on them for vector listing
	2170	+ //error_log('Pinecone index stats: ' . print_r($stats_data, true));
3882	2171	}
3883	2172	}
3884		-
	2173	+
3885	2174	} catch (Exception $e) {
3886		- // Log error but return fresh data only
	2175	+ //error_log('Pinecone processed content exception: ' . $e->getMessage());
3887	2176	}
3888		-
3889		- return $pinecone_data;
	2177	+
	2178	+ // Merge cached data with Pinecone data
	2179	+ // Cache takes priority for recent updates (within last 5 minutes)
	2180	+ $merged_data = $pinecone_data;
	2181	+
	2182	+ foreach ($cached_data as $post_id => $cache_item) {
	2183	+ $cache_timestamp = $cache_item['timestamp'] ?? 0;
	2184	+ $time_diff = current_time('timestamp') - $cache_timestamp;
	2185	+
	2186	+ // If cache item is recent (less than 5 minutes), prioritize it
	2187	+ if ($time_diff < 300) { // 5 minutes = 300 seconds
	2188	+ $merged_data[$post_id] = $cache_item;
	2189	+ } else {
	2190	+ // If not in Pinecone data and cache is old, keep cache but mark as potentially stale
	2191	+ if (!isset($merged_data[$post_id])) {
	2192	+ $merged_data[$post_id] = $cache_item;
	2193	+ }
	2194	+ }
	2195	+ }
	2196	+
	2197	+ return $merged_data;
3890	2198	}
3891	2199	public function mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $vector_ids) {
3892		- //error_log('=== DEBUG: Starting mxchat_fetch_pinecone_vectors_by_ids ===');
3893		-
3894	2200	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
3895	2201	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
3896		-
	2202	+
3897	2203	if (empty($api_key) || empty($host) || empty($vector_ids)) {
3898		- //error_log('DEBUG: Missing parameters for fetch by IDs');
3899	2204	return array();
3900	2205	}
3901		-
	2206	+
3902	2207	try {
3903	2208	$fetch_url = "https://{$host}/vectors/fetch";
3904		- //error_log('DEBUG: Fetch URL: ' . $fetch_url);
3905		- //error_log('DEBUG: Fetching ' . count($vector_ids) . ' vector IDs');
3906		-
	2209	+
3907	2210	// Pinecone fetch API allows fetching specific vectors by ID
3908	2211	$fetch_data = array(
3909	2212	'ids' => array_values($vector_ids)
3910	2213	);
3911		-
	2214	+
3912	2215	$response = wp_remote_post($fetch_url, array(
3913	2216	'headers' => array(
3914	2217	'Api-Key' => $api_key,
3915	2218	'Content-Type' => 'application/json'
		@@ -3916,45 +2219,39 @@
3916	2219	),
3917	2220	'body' => json_encode($fetch_data),
3918	2221	'timeout' => 30
3919	2222	));
3920		-
	2223	+
3921	2224	if (is_wp_error($response)) {
3922		- //error_log('DEBUG: Fetch by IDs WP error: ' . $response->get_error_message());
	2225	+ //error_log('Pinecone fetch by IDs error: ' . $response->get_error_message());
3923	2226	return array();
3924	2227	}
3925		-
	2228	+
3926	2229	$response_code = wp_remote_retrieve_response_code($response);
3927		- //error_log('DEBUG: Fetch response code: ' . $response_code);
3928		-
3929	2230	if ($response_code !== 200) {
3930		- $error_body = wp_remote_retrieve_body($response);
3931		- //error_log('DEBUG: Fetch failed with body: ' . $error_body);
	2231	+ //error_log('Pinecone fetch by IDs failed with code: ' . $response_code);
3932	2232	return array();
3933	2233	}
3934		-
	2234	+
3935	2235	$body = wp_remote_retrieve_body($response);
3936	2236	$data = json_decode($body, true);
3937	2237
3938		- //error_log('DEBUG: Fetch response structure: ' . print_r(array_keys($data), true));
3939		-
3940	2238	if (!isset($data['vectors'])) {
3941		- //error_log('DEBUG: No vectors key in response');
3942	2239	return array();
3943	2240	}
3944		-
	2241	+
3945	2242	$processed_data = array();
3946		-
	2243	+
3947	2244	foreach ($data['vectors'] as $vector_id => $vector_data) {
3948	2245	$metadata = $vector_data['metadata'] ?? array();
3949	2246	$source_url = $metadata['source_url'] ?? '';
3950		-
	2247	+
3951	2248	if (!empty($source_url)) {
3952	2249	$post_id = url_to_postid($source_url);
3953	2250	if ($post_id) {
3954	2251	$created_at = $metadata['created_at'] ?? '';
3955		- $processed_date = 'Recently';
3956		-
	2252	+ $processed_date = 'Recently'; // Default
	2253	+
3957	2254	if (!empty($created_at)) {
3958	2255	$timestamp = is_numeric($created_at) ? $created_at : strtotime($created_at);
3959	2256	if ($timestamp) {
3960	2257	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
		@@ -3959,9 +2256,9 @@
3959	2256	if ($timestamp) {
3960	2257	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
3961	2258	}
3962	2259	}
3963		-
	2260	+
3964	2261	$processed_data[$post_id] = array(
3965	2262	'db_id' => $vector_id,
3966	2263	'processed_date' => $processed_date,
3967	2264	'url' => $source_url,
		@@ -3970,90 +2267,52 @@
3970	2267	);
3971	2268	}
3972	2269	}
3973	2270	}
3974		-
3975		- //error_log('DEBUG: Processed ' . count($processed_data) . ' vectors from fetch');
	2271	+
3976	2272	return $processed_data;
3977		-
	2273	+
3978	2274	} catch (Exception $e) {
3979		- //error_log('DEBUG: Exception in fetch_pinecone_vectors_by_ids: ' . $e->getMessage());
	2275	+ //error_log('Pinecone fetch by IDs exception: ' . $e->getMessage());
3980	2276	return array();
3981	2277	}
3982	2278	}
3983		-
3984		-/**
3985		- * Get embedding dimensions based on the selected model.
3986		- */
3987		-private function mxchat_get_embedding_dimensions() {
3988		- $options = get_option('mxchat_options', array());
3989		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
3990		-
3991		- $model_dimensions = array(
3992		- 'text-embedding-ada-002' => 1536,
3993		- 'text-embedding-3-small' => 1536,
3994		- 'text-embedding-3-large' => 3072,
3995		- 'voyage-2' => 1024,
3996		- 'voyage-large-2' => 1536,
3997		- 'voyage-3-large' => 2048,
3998		- 'gemini-embedding-001' => 1536,
3999		- );
4000		-
4001		- if (strpos($selected_model, 'voyage-3-large') === 0) {
4002		- $custom_dimensions = $options['voyage_output_dimension'] ?? 2048;
4003		- return intval($custom_dimensions);
4004		- }
4005		-
4006		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4007		- $custom_dimensions = $options['gemini_output_dimension'] ?? 1536;
4008		- return intval($custom_dimensions);
4009		- }
4010		-
4011		- return $model_dimensions[$selected_model] ?? 1536;
4012		-}
4013		-
4014		-/**
4015		- * Scan Pinecone for processed content
4016		- */
4017	2279	public function mxchat_scan_pinecone_for_processed_content($pinecone_options) {
4018	2280	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4019	2281	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
4020		-
	2282	+
4021	2283	if (empty($api_key) || empty($host)) {
4022	2284	return array();
4023	2285	}
4024		-
	2286	+
4025	2287	try {
4026	2288	// Use multiple random vectors to get better coverage
4027	2289	$all_matches = array();
4028	2290	$seen_ids = array();
4029		-
4030		- // Get correct dimensions for the configured embedding model
4031		- $dimensions = $this->mxchat_get_embedding_dimensions();
4032		-
	2291	+
4033	2292	// Try 3 different random vectors to get better coverage
4034	2293	for ($i = 0; $i < 3; $i++) {
4035	2294	$query_url = "https://{$host}/query";
4036		-
	2295	+
4037	2296	// Generate a random unit vector instead of zeros
4038	2297	$random_vector = array();
4039		- for ($j = 0; $j < $dimensions; $j++) {
4040		- $random_vector[] = (rand(-1000, 1000) / 1000.0);
	2298	+ for ($j = 0; $j < 1536; $j++) {
	2299	+ $random_vector[] = (rand(-1000, 1000) / 1000.0); // Random values between -1 and 1
4041	2300	}
4042		-
	2301	+
4043	2302	// Normalize the vector to unit length
4044	2303	$magnitude = sqrt(array_sum(array_map(function($x) { return $x * $x; }, $random_vector)));
4045	2304	if ($magnitude > 0) {
4046	2305	$random_vector = array_map(function($x) use ($magnitude) { return $x / $magnitude; }, $random_vector);
4047	2306	}
4048		-
	2307	+
4049	2308	$query_data = array(
4050	2309	'includeMetadata' => true,
4051	2310	'includeValues' => false,
4052		- 'topK' => 10000,
	2311	+ 'topK' => 10000, // Get many results
4053	2312	'vector' => $random_vector
4054	2313	);
4055		-
	2314	+
4056	2315	$response = wp_remote_post($query_url, array(
4057	2316	'headers' => array(
4058	2317	'Api-Key' => $api_key,
4059	2318	'Content-Type' => 'application/json'
		@@ -4060,22 +2319,21 @@
4060	2319	),
4061	2320	'body' => json_encode($query_data),
4062	2321	'timeout' => 30
4063	2322	));
4064		-
	2323	+
4065	2324	if (is_wp_error($response)) {
4066	2325	continue;
4067	2326	}
4068		-
	2327	+
4069	2328	$response_code = wp_remote_retrieve_response_code($response);
4070		-
4071	2329	if ($response_code !== 200) {
4072	2330	continue;
4073	2331	}
4074		-
	2332	+
4075	2333	$body = wp_remote_retrieve_body($response);
4076	2334	$data = json_decode($body, true);
4077		-
	2335	+
4078	2336	if (isset($data['matches'])) {
4079	2337	foreach ($data['matches'] as $match) {
4080	2338	$match_id = $match['id'] ?? '';
4081	2339	if (!empty($match_id) && !isset($seen_ids[$match_id])) {
		@@ -4084,30 +2342,24 @@
4084	2342	}
4085	2343	}
4086	2344	}
4087	2345	}
4088		-
4089		- // Convert matches to processed data format, grouping by URL to count chunks
	2346	+
	2347	+ // Convert matches to processed data format
4090	2348	$processed_data = array();
4091		- $url_chunk_counts = array();
4092		-
	2349	+ $vector_ids_for_cache = array();
	2350	+
4093	2351	foreach ($all_matches as $match) {
4094	2352	$metadata = $match['metadata'] ?? array();
4095	2353	$source_url = $metadata['source_url'] ?? '';
4096	2354	$match_id = $match['id'] ?? '';
4097		-
	2355	+
4098	2356	if (!empty($source_url) && !empty($match_id)) {
4099	2357	$post_id = url_to_postid($source_url);
4100	2358	if ($post_id) {
4101		- // Count chunks per post_id
4102		- if (!isset($url_chunk_counts[$post_id])) {
4103		- $url_chunk_counts[$post_id] = 0;
4104		- }
4105		- $url_chunk_counts[$post_id]++;
4106		-
4107	2359	$created_at = $metadata['created_at'] ?? '';
4108		- $processed_date = 'Recently';
4109		-
	2360	+ $processed_date = 'Recently'; // Default
	2361	+
4110	2362	if (!empty($created_at)) {
4111	2363	$timestamp = is_numeric($created_at) ? $created_at : strtotime($created_at);
4112	2364	if ($timestamp) {
4113	2365	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
		@@ -4112,312 +2364,242 @@
4112	2364	if ($timestamp) {
4113	2365	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
4114	2366	}
4115	2367	}
4116		-
4117		- // Only store if not already set, or update with newer timestamp
4118		- if (!isset($processed_data[$post_id]) ||
4119		- ($timestamp ?? 0) > ($processed_data[$post_id]['timestamp'] ?? 0)) {
4120		- $processed_data[$post_id] = array(
4121		- 'db_id' => $match_id,
4122		- 'processed_date' => $processed_date,
4123		- 'url' => $source_url,
4124		- 'source' => 'pinecone',
4125		- 'timestamp' => $timestamp ?? current_time('timestamp')
4126		- );
4127		- }
	2368	+
	2369	+ $processed_data[$post_id] = array(
	2370	+ 'db_id' => $match_id,
	2371	+ 'processed_date' => $processed_date,
	2372	+ 'url' => $source_url,
	2373	+ 'source' => 'pinecone',
	2374	+ 'timestamp' => $timestamp ?? current_time('timestamp')
	2375	+ );
	2376	+
	2377	+ $vector_ids_for_cache[] = $match_id;
4128	2378	}
4129	2379	}
4130	2380	}
4131		-
4132		- // Add chunk counts to processed data
4133		- foreach ($url_chunk_counts as $post_id => $chunk_count) {
4134		- if (isset($processed_data[$post_id])) {
4135		- $processed_data[$post_id]['chunk_count'] = $chunk_count;
4136		- }
	2381	+
	2382	+ // Update the vector IDs cache for future use
	2383	+ if (!empty($vector_ids_for_cache)) {
	2384	+ update_option('mxchat_pinecone_vector_ids_cache', $vector_ids_for_cache);
4137	2385	}
4138		-
	2386	+
4139	2387	return $processed_data;
4140		-
	2388	+
4141	2389	} catch (Exception $e) {
	2390	+ //error_log('Pinecone scan exception: ' . $e->getMessage());
4142	2391	return array();
4143	2392	}
4144		-}
4145		-/**
4146		- * Generate embeddings from input text for MXChat with bot support
4147		- */
4148		-private function mxchat_generate_embedding($text, $bot_id = 'default') {
4149		- // Enable detailed logging for debugging
4150		- //error_log('[MXCHAT-EMBED] Starting embedding generation for bot: ' . $bot_id . '. Text length: ' . strlen($text) . ' bytes');
4151		- //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
	2393	+}
4152	2394
4153		- // Get bot-specific options
4154		- $bot_options = $this->get_bot_options($bot_id);
4155		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
4156		-
4157		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4158		- //error_log('[MXCHAT-EMBED] Selected embedding model for bot ' . $bot_id . ': ' . $selected_model);
	2395	+ /**
	2396	+ * Generates embeddings from input text for MXChat
	2397	+ */
	2398	+ private function mxchat_generate_embedding($text) {
	2399	+ // Enable detailed logging for debugging
	2400	+ //error_log('[MXCHAT-EMBED] Starting embedding generation. Text length: ' . strlen($text) . ' bytes');
	2401	+ //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4159	2402
4160		- // Determine provider and endpoint
4161		- if (strpos($selected_model, 'voyage') === 0) {
4162		- $api_key = $options['voyage_api_key'] ?? '';
4163		- $endpoint = 'https://api.voyageai.com/v1/embeddings';
4164		- $provider_name = 'Voyage AI';
4165		- //error_log('[MXCHAT-EMBED] Using Voyage AI API for bot ' . $bot_id);
4166		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
4167		- $api_key = $options['gemini_api_key'] ?? '';
4168		- $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
4169		- $provider_name = 'Google Gemini';
4170		- //error_log('[MXCHAT-EMBED] Using Google Gemini API for bot ' . $bot_id);
4171		- } else {
4172		- $api_key = $options['api_key'] ?? '';
4173		- $endpoint = 'https://api.openai.com/v1/embeddings';
4174		- $provider_name = 'OpenAI';
4175		- //error_log('[MXCHAT-EMBED] Using OpenAI API for bot ' . $bot_id);
4176		- }
	2403	+ $options = get_option('mxchat_options');
	2404	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	2405	+ //error_log('[MXCHAT-EMBED] Selected embedding model: ' . $selected_model);
4177	2406
4178		- //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
	2407	+ // Determine provider and endpoint
	2408	+ if (strpos($selected_model, 'voyage') === 0) {
	2409	+ $api_key = $options['voyage_api_key'] ?? '';
	2410	+ $endpoint = 'https://api.voyageai.com/v1/embeddings';
	2411	+ $provider_name = 'Voyage AI';
	2412	+ //error_log('[MXCHAT-EMBED] Using Voyage AI API');
	2413	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	2414	+ $api_key = $options['gemini_api_key'] ?? '';
	2415	+ $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
	2416	+ $provider_name = 'Google Gemini';
	2417	+ //error_log('[MXCHAT-EMBED] Using Google Gemini API');
	2418	+ } else {
	2419	+ $api_key = $options['api_key'] ?? '';
	2420	+ $endpoint = 'https://api.openai.com/v1/embeddings';
	2421	+ $provider_name = 'OpenAI';
	2422	+ //error_log('[MXCHAT-EMBED] Using OpenAI API');
	2423	+ }
4179	2424
4180		- if (empty($api_key)) {
4181		- $error_message = sprintf('Missing %s API key for bot %s. Please configure your API key in the bot settings.', $provider_name, $bot_id);
4182		- //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
4183		- return $error_message;
4184		- }
	2425	+ //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
4185	2426
4186		- // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
4187		- $estimated_tokens = ceil(str_word_count($text) / 0.75);
4188		- //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
	2427	+ if (empty($api_key)) {
	2428	+ $error_message = sprintf('Missing %s API key. Please configure your API key in the MxChat settings.', $provider_name);
	2429	+ //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
	2430	+ return $error_message;
	2431	+ }
4189	2432
4190		- if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
4191		- //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
4192		- // Consider truncating text here
4193		- }
	2433	+ // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
	2434	+ $estimated_tokens = ceil(str_word_count($text) / 0.75);
	2435	+ //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
4194	2436
4195		- // Prepare request body based on provider
4196		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4197		- // Gemini API format
4198		- $request_body = array(
4199		- 'model' => 'models/' . $selected_model,
4200		- 'content' => array(
4201		- 'parts' => array(
4202		- array('text' => $text)
4203		- )
4204		- )
4205		- );
	2437	+ if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
	2438	+ //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
	2439	+ // Consider truncating text here
	2440	+ }
4206	2441
4207		- // Set output dimensionality to 1536 for consistency with other models
4208		- $request_body['outputDimensionality'] = 1536;
4209		- } else {
4210		- // OpenAI/Voyage API format
4211		- $request_body = array(
4212		- 'model' => $selected_model,
4213		- 'input' => $text
4214		- );
	2442	+ // Prepare request body based on provider
	2443	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2444	+ // Gemini API format
	2445	+ $request_body = array(
	2446	+ 'model' => 'models/' . $selected_model,
	2447	+ 'content' => array(
	2448	+ 'parts' => array(
	2449	+ array('text' => $text)
	2450	+ )
	2451	+ )
	2452	+ );
4215	2453
4216		- // Add output_dimension for voyage-3-large model
4217		- if ($selected_model === 'voyage-3-large') {
4218		- $request_body['output_dimension'] = 2048;
4219		- }
4220		- }
	2454	+ // Set output dimensionality to 1536 for consistency with other models
	2455	+ $request_body['outputDimensionality'] = 1536;
	2456	+ } else {
	2457	+ // OpenAI/Voyage API format
	2458	+ $request_body = array(
	2459	+ 'model' => $selected_model,
	2460	+ 'input' => $text
	2461	+ );
4221	2462
4222		- //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
	2463	+ // Add output_dimension for voyage-3-large model
	2464	+ if ($selected_model === 'voyage-3-large') {
	2465	+ $request_body['output_dimension'] = 2048;
	2466	+ }
	2467	+ }
4223	2468
4224		- // Prepare headers based on provider
4225		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4226		- // Gemini uses API key as query parameter
4227		- $endpoint .= '?key=' . $api_key;
4228		- $headers = array(
4229		- 'Content-Type' => 'application/json'
4230		- );
4231		- } else {
4232		- // OpenAI/Voyage use Bearer token
4233		- $headers = array(
4234		- 'Authorization' => 'Bearer ' . $api_key,
4235		- 'Content-Type' => 'application/json'
4236		- );
4237		- }
	2469	+ //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
4238	2470
4239		- // Make API request
4240		- //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
4241		- $response = wp_remote_post($endpoint, array(
4242		- 'body' => wp_json_encode($request_body),
4243		- 'headers' => $headers,
4244		- 'timeout' => 60 // Increased timeout for large inputs
4245		- ));
	2471	+ // Prepare headers based on provider
	2472	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2473	+ // Gemini uses API key as query parameter
	2474	+ $endpoint .= '?key=' . $api_key;
	2475	+ $headers = array(
	2476	+ 'Content-Type' => 'application/json'
	2477	+ );
	2478	+ } else {
	2479	+ // OpenAI/Voyage use Bearer token
	2480	+ $headers = array(
	2481	+ 'Authorization' => 'Bearer ' . $api_key,
	2482	+ 'Content-Type' => 'application/json'
	2483	+ );
	2484	+ }
4246	2485
4247		- // Handle wp_remote_post errors
4248		- if (is_wp_error($response)) {
4249		- $error_message = $response->get_error_message();
4250		- //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
4251		- return 'Connection error: ' . $error_message;
4252		- }
	2486	+ // Make API request
	2487	+ //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
	2488	+ $response = wp_remote_post($endpoint, array(
	2489	+ 'body' => wp_json_encode($request_body),
	2490	+ 'headers' => $headers,
	2491	+ 'timeout' => 60 // Increased timeout for large inputs
	2492	+ ));
4253	2493
4254		- // Get and check HTTP response code
4255		- $http_code = wp_remote_retrieve_response_code($response);
4256		- //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
	2494	+ // Handle wp_remote_post errors
	2495	+ if (is_wp_error($response)) {
	2496	+ $error_message = $response->get_error_message();
	2497	+ //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
	2498	+ return 'Connection error: ' . $error_message;
	2499	+ }
4257	2500
4258		- if ($http_code !== 200) {
4259		- $error_body = wp_remote_retrieve_body($response);
4260		- //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
	2501	+ // Get and check HTTP response code
	2502	+ $http_code = wp_remote_retrieve_response_code($response);
	2503	+ //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
4261	2504
4262		- // Try to parse error for more details
4263		- $error_json = json_decode($error_body, true);
4264		- if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
4265		- $error_type = $error_json['error']['type'] ?? 'unknown';
4266		- $error_message = $error_json['error']['message'] ?? 'No message';
4267		- //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
4268		- //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
	2505	+ if ($http_code !== 200) {
	2506	+ $error_body = wp_remote_retrieve_body($response);
	2507	+ //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
4269	2508
4270		- // Customize error message for common API errors
4271		- if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
4272		- $error_message = sprintf('Invalid %s API key for bot %s. Please check your API key in the bot settings.', $provider_name, $bot_id);
4273		- } elseif ($error_type === 'authentication_error') {
4274		- $error_message = sprintf('%s authentication failed for bot %s. Please verify your API key in the bot settings.', $provider_name, $bot_id);
4275		- }
	2509	+ // Try to parse error for more details
	2510	+ $error_json = json_decode($error_body, true);
	2511	+ if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
	2512	+ $error_type = $error_json['error']['type'] ?? 'unknown';
	2513	+ $error_message = $error_json['error']['message'] ?? 'No message';
	2514	+ //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
	2515	+ //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
4276	2516
4277		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4278		- return $error_message;
4279		- }
	2517	+ // Customize error message for common API errors
	2518	+ if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
	2519	+ $error_message = sprintf('Invalid %s API key. Please check your API key in the MxChat settings.', $provider_name);
	2520	+ } elseif ($error_type === 'authentication_error') {
	2521	+ $error_message = sprintf('%s authentication failed. Please verify your API key in the MxChat settings.', $provider_name);
	2522	+ }
4280	2523
4281		- $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding for bot %s", $http_code, $bot_id);
4282		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4283		- return $error_message;
4284		- }
	2524	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2525	+ return $error_message;
	2526	+ }
4285	2527
4286		- // Parse response body
4287		- $response_body = wp_remote_retrieve_body($response);
4288		- //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
	2528	+ $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding", $http_code);
	2529	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2530	+ return $error_message;
	2531	+ }
4289	2532
4290		- $response_data = json_decode($response_body, true);
	2533	+ // Parse response body
	2534	+ $response_body = wp_remote_retrieve_body($response);
	2535	+ //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
4291	2536
4292		- if (json_last_error() !== JSON_ERROR_NONE) {
4293		- $error = json_last_error_msg();
4294		- //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
4295		- //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
4296		- return "Failed to parse API response: $error";
4297		- }
	2537	+ $response_data = json_decode($response_body, true);
4298	2538
4299		- // Handle different response formats based on provider
4300		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4301		- // Gemini API response format
4302		- if (isset($response_data['embedding']['values'])) {
4303		- $embedding_dimensions = count($response_data['embedding']['values']);
4304		- //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
	2539	+ if (json_last_error() !== JSON_ERROR_NONE) {
	2540	+ $error = json_last_error_msg();
	2541	+ //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
	2542	+ //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
	2543	+ return "Failed to parse API response: $error";
	2544	+ }
4305	2545
4306		- // Check if embedding dimensions are as expected (should be 1536)
4307		- if ($embedding_dimensions !== 1536) {
4308		- //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
4309		- }
	2546	+ // Handle different response formats based on provider
	2547	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2548	+ // Gemini API response format
	2549	+ if (isset($response_data['embedding']['values'])) {
	2550	+ $embedding_dimensions = count($response_data['embedding']['values']);
	2551	+ //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
4310	2552
4311		- return $response_data['embedding']['values'];
4312		- } else {
4313		- //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
4314		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2553	+ // Check if embedding dimensions are as expected (should be 1536)
	2554	+ if ($embedding_dimensions !== 1536) {
	2555	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
	2556	+ }
4315	2557
4316		- if (isset($response_data['error'])) {
4317		- $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
4318		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4319		- return $error_message;
4320		- }
	2558	+ return $response_data['embedding']['values'];
	2559	+ } else {
	2560	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
	2561	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4321	2562
4322		- $error_message = "Invalid Gemini API response format: No embedding found";
4323		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4324		- return $error_message;
4325		- }
4326		- } else {
4327		- // OpenAI/Voyage API response format
4328		- if (isset($response_data['data'][0]['embedding'])) {
4329		- $embedding_dimensions = count($response_data['data'][0]['embedding']);
4330		- //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
	2563	+ if (isset($response_data['error'])) {
	2564	+ $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
	2565	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2566	+ return $error_message;
	2567	+ }
4331	2568
4332		- // Check if embedding dimensions are as expected
4333		- if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
4334		- ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
4335		- //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
4336		- }
	2569	+ $error_message = "Invalid Gemini API response format: No embedding found";
	2570	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2571	+ return $error_message;
	2572	+ }
	2573	+ } else {
	2574	+ // OpenAI/Voyage API response format
	2575	+ if (isset($response_data['data'][0]['embedding'])) {
	2576	+ $embedding_dimensions = count($response_data['data'][0]['embedding']);
	2577	+ //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
4337	2578
4338		- return $response_data['data'][0]['embedding'];
4339		- } else {
4340		- //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
4341		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2579	+ // Check if embedding dimensions are as expected
	2580	+ if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
	2581	+ ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
	2582	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
	2583	+ }
4342	2584
4343		- if (isset($response_data['error'])) {
4344		- $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
4345		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4346		- return $error_message;
4347		- }
	2585	+ return $response_data['data'][0]['embedding'];
	2586	+ } else {
	2587	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
	2588	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4348	2589
4349		- $error_message = "Invalid API response format: No embedding found";
4350		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4351		- return $error_message;
4352		- }
4353		- }
4354		-}
	2590	+ if (isset($response_data['error'])) {
	2591	+ $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
	2592	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2593	+ return $error_message;
	2594	+ }
4355	2595
4356		-/**
4357		- * Get bot-specific options for multi-bot functionality
4358		- * Falls back to default options if bot_id is 'default' or multi-bot add-on is not active
4359		- */
4360		-private function get_bot_options($bot_id = 'default') {
4361		- //error_log("MXCHAT DEBUG: get_bot_options called for bot: " . $bot_id);
4362		-
4363		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4364		- //error_log("MXCHAT DEBUG: Using default options (no multi-bot or bot is 'default')");
4365		- return array();
4366		- }
4367		-
4368		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
4369		-
4370		- if (!empty($bot_options)) {
4371		- //error_log("MXCHAT DEBUG: Got bot-specific options from filter");
4372		- if (isset($bot_options['similarity_threshold'])) {
4373		- //error_log(" - similarity_threshold: " . $bot_options['similarity_threshold']);
4374		- }
4375		- }
4376		-
4377		- return is_array($bot_options) ? $bot_options : array();
4378		-}
4379		-
4380		-/**
4381		- * Get bot-specific Pinecone configuration
4382		- * Used in the knowledge retrieval functions
4383		- */
4384		-// Also add debugging to your get_bot_pinecone_config function
4385		-private function get_bot_pinecone_config($bot_id = 'default') {
4386		- //error_log("MXCHAT DEBUG: get_bot_pinecone_config called for bot: " . $bot_id);
4387		-
4388		- // If default bot or multi-bot add-on not active, use default Pinecone config
4389		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4390		- //error_log("MXCHAT DEBUG: Using default Pinecone config (no multi-bot or bot is 'default')");
4391		- $addon_options = get_option('mxchat_pinecone_addon_options', array());
4392		- $config = array(
4393		- 'use_pinecone' => (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1'),
4394		- 'api_key' => $addon_options['mxchat_pinecone_api_key'] ?? '',
4395		- 'host' => $addon_options['mxchat_pinecone_host'] ?? '',
4396		- 'namespace' => $addon_options['mxchat_pinecone_namespace'] ?? ''
4397		- );
4398		- //error_log("MXCHAT DEBUG: Default config - use_pinecone: " . ($config['use_pinecone'] ? 'true' : 'false'));
4399		- return $config;
4400		- }
4401		-
4402		- //error_log("MXCHAT DEBUG: Calling filter 'mxchat_get_bot_pinecone_config' for bot: " . $bot_id);
4403		-
4404		- // Hook for multi-bot add-on to provide bot-specific Pinecone config
4405		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
4406		-
4407		- if (!empty($bot_pinecone_config)) {
4408		- //error_log("MXCHAT DEBUG: Got bot-specific config from filter");
4409		- //error_log(" - use_pinecone: " . (isset($bot_pinecone_config['use_pinecone']) ? ($bot_pinecone_config['use_pinecone'] ? 'true' : 'false') : 'not set'));
4410		- //error_log(" - host: " . ($bot_pinecone_config['host'] ?? 'not set'));
4411		- //error_log(" - namespace: " . ($bot_pinecone_config['namespace'] ?? 'not set'));
4412		- } else {
4413		- //error_log("MXCHAT DEBUG: Filter returned empty config!");
4414		- }
4415		-
4416		- return is_array($bot_pinecone_config) ? $bot_pinecone_config : array();
4417		-}
4418		-
4419		-
	2596	+ $error_message = "Invalid API response format: No embedding found";
	2597	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2598	+ return $error_message;
	2599	+ }
	2600	+ }
	2601	+ }
4420	2602	public function mxchat_ajax_dismiss_completed_status() {
4421	2603	try {
4422	2604	// Verify the request
4423	2605	check_ajax_referer('mxchat_status_nonce', 'nonce');
		@@ -4536,10 +2718,10 @@
4536	2718
4537	2719	$html .= '<p><strong>' . esc_html__('Status:', 'mxchat') . '</strong> ' . esc_html(ucfirst($status['status'])) . '</p>';
4538	2720	$html .= '<p><strong>' . esc_html__('Last update:', 'mxchat') . '</strong> ' . esc_html($status['last_update']) . '</p>';
4539	2721
4540		- // Add completion summary if available AND it's an array
4541		- if (isset($status['completion_summary']) && is_array($status['completion_summary']) && !empty($status['completion_summary'])) {
	2722	+ // Add completion summary if available
	2723	+ if (isset($status['completion_summary'])) {
4542	2724	$summary = $status['completion_summary'];
4543	2725	$html .= '<div class="mxchat-completion-summary">';
4544	2726	$html .= '<h5>' . esc_html__('Processing Summary', 'mxchat') . '</h5>';
4545	2727	$html .= '<p><strong>' . esc_html__('Total Pages:', 'mxchat') . '</strong> ' . esc_html($summary['total_pages']) . '</p>';
		@@ -4548,10 +2730,10 @@
4548	2730	$html .= '<p><strong>' . esc_html__('Completed:', 'mxchat') . '</strong> ' . esc_html($summary['completion_time']) . '</p>';
4549	2731	$html .= '</div>';
4550	2732	}
4551	2733
4552		- // Add failed pages list if any AND it's an array
4553		- if (isset($status['failed_pages_list']) && is_array($status['failed_pages_list']) && !empty($status['failed_pages_list'])) {
	2734	+ // Add failed pages list if any
	2735	+ if (!empty($status['failed_pages_list'])) {
4554	2736	$html .= $this->mxchat_render_failed_pages_list($status['failed_pages_list']);
4555	2737	}
4556	2738
4557	2739	// Add error message if any
		@@ -4565,8 +2747,9 @@
4565	2747	$html .= '</div>'; // End card
4566	2748
4567	2749	return $html;
4568	2750	}
	2751	+
4569	2752	/**
4570	2753	* Render sitemap status card HTML
4571	2754	*/
4572	2755	private function mxchat_render_sitemap_status_card($status, $sitemap_url) {
		@@ -4622,10 +2805,10 @@
4622	2805
4623	2806	$html .= '<p><strong>' . esc_html__('Status:', 'mxchat') . '</strong> ' . esc_html(ucfirst($status['status'])) . '</p>';
4624	2807	$html .= '<p><strong>' . esc_html__('Last update:', 'mxchat') . '</strong> ' . esc_html($status['last_update']) . '</p>';
4625	2808
4626		- // Add completion summary if available AND it's an array
4627		- if (isset($status['completion_summary']) && is_array($status['completion_summary']) && !empty($status['completion_summary'])) {
	2809	+ // Add completion summary if available
	2810	+ if (isset($status['completion_summary'])) {
4628	2811	$summary = $status['completion_summary'];
4629	2812	$html .= '<div class="mxchat-completion-summary">';
4630	2813	$html .= '<h5>' . esc_html__('Processing Summary', 'mxchat') . '</h5>';
4631	2814	$html .= '<p><strong>' . esc_html__('Total URLs:', 'mxchat') . '</strong> ' . esc_html($summary['total_urls']) . '</p>';
		@@ -4654,16 +2837,13 @@
4654	2837	$html .= '</div>'; // End card
4655	2838
4656	2839	return $html;
4657	2840	}
4658		-
4659		-
4660	2841	/**
4661	2842	* Render failed pages list
4662	2843	*/
4663	2844	private function mxchat_render_failed_pages_list($failed_pages_list) {
4664		- // Validate that $failed_pages_list is an array and not empty
4665		- if (!is_array($failed_pages_list) || empty($failed_pages_list)) {
	2845	+ if (empty($failed_pages_list)) {
4666	2846	return '';
4667	2847	}
4668	2848
4669	2849	$html = '<div class="mxchat-error-notice">';
		@@ -4688,13 +2868,8 @@
4688	2868	return ($b['time'] ?? 0) - ($a['time'] ?? 0);
4689	2869	});
4690	2870
4691	2871	foreach ($sorted_failed_pages as $item) {
4692		- // Ensure $item is an array before accessing its elements
4693		- if (!is_array($item)) {
4694		- continue;
4695		- }
4696		-
4697	2872	$time_ago = isset($item['time']) ? human_time_diff($item['time'], current_time('timestamp')) . ' ' . esc_html__('ago', 'mxchat') : 'Unknown';
4698	2873	$html .= '<tr>';
4699	2874	$html .= '<td>' . esc_html__('Page', 'mxchat') . ' ' . esc_html($item['page'] ?? 'Unknown') . '</td>';
4700	2875	$html .= '<td style="word-break: break-word;">' . esc_html($item['error'] ?? 'Unknown error') . '</td>';
		@@ -4712,10 +2887,9 @@
4712	2887	/**
4713	2888	* Render failed URLs list
4714	2889	*/
4715	2890	private function mxchat_render_failed_urls_list($failed_urls_list) {
4716		- // Validate that $failed_urls_list is an array and not empty
4717		- if (!is_array($failed_urls_list) || empty($failed_urls_list)) {
	2891	+ if (empty($failed_urls_list)) {
4718	2892	return '';
4719	2893	}
4720	2894
4721	2895	$html = '<div class="mxchat-failed-urls-container">';
		@@ -4742,13 +2916,8 @@
4742	2916	// Show up to 50 failed URLs
4743	2917	$display_urls = array_slice($sorted_failed_urls, 0, 50);
4744	2918
4745	2919	foreach ($display_urls as $item) {
4746		- // Ensure $item is an array before accessing its elements
4747		- if (!is_array($item)) {
4748		- continue;
4749		- }
4750		-
4751	2920	$url = $item['url'] ?? '';
4752	2921	$time_ago = isset($item['time']) ? human_time_diff($item['time'], current_time('timestamp')) . ' ' . esc_html__('ago', 'mxchat') : 'Unknown';
4753	2922
4754	2923	// Truncate URL for display
		@@ -4780,375 +2949,18 @@
4780	2949
4781	2950	return $html;
4782	2951	}
4783	2952
4784		-/**
4785		- * Get all ACF fields for a specific post, excluding any fields the user has disabled
4786		- */
4787		-public function mxchat_get_acf_fields_for_post($post_id) {
4788		- if (!function_exists('get_fields')) {
4789		- return array();
4790		- }
4791		-
4792		- $fields = get_fields($post_id);
4793		- if (!$fields || !is_array($fields)) {
4794		- return array();
4795		- }
4796		-
4797		- // Get excluded fields from settings
4798		- $excluded_fields = get_option('mxchat_acf_excluded_fields', array());
4799		- if (!empty($excluded_fields) && is_array($excluded_fields)) {
4800		- foreach ($excluded_fields as $excluded_field) {
4801		- if (isset($fields[$excluded_field])) {
4802		- unset($fields[$excluded_field]);
4803		- }
4804		- }
4805		- }
4806		-
4807		- return $fields;
4808		-}
4809		-
4810		-/**
4811		- * Get all registered ACF field groups and their fields for the settings UI
4812		- */
4813		-public function mxchat_get_all_acf_fields() {
4814		- if (!function_exists('acf_get_field_groups') || !function_exists('acf_get_fields')) {
4815		- return array();
4816		- }
4817		-
4818		- $all_fields = array();
4819		- $field_groups = acf_get_field_groups();
4820		-
4821		- if (!empty($field_groups)) {
4822		- foreach ($field_groups as $group) {
4823		- $group_fields = acf_get_fields($group['key']);
4824		- if (!empty($group_fields)) {
4825		- $all_fields[$group['title']] = array();
4826		- foreach ($group_fields as $field) {
4827		- $all_fields[$group['title']][] = array(
4828		- 'name' => $field['name'],
4829		- 'label' => $field['label'],
4830		- 'type' => $field['type']
4831		- );
4832		- }
4833		- }
4834		- }
4835		- }
4836		-
4837		- return $all_fields;
4838		-}
4839		-
4840		-/**
4841		- * Get whitelisted custom post meta for a given post
4842		- * This allows non-ACF meta fields (like OptionTree, theme meta boxes) to be included in embeddings
4843		- */
4844		-public function mxchat_get_whitelisted_post_meta($post_id) {
4845		- $whitelist = get_option('mxchat_custom_meta_whitelist', '');
4846		-
4847		- if (empty($whitelist)) {
4848		- return array();
4849		- }
4850		-
4851		- // Parse the whitelist - one meta key per line
4852		- $meta_keys = array_filter(array_map('trim', explode("\n", $whitelist)));
4853		-
4854		- if (empty($meta_keys)) {
4855		- return array();
4856		- }
4857		-
4858		- $result = array();
4859		-
4860		- foreach ($meta_keys as $key) {
4861		- // Skip empty keys
4862		- if (empty($key)) {
4863		- continue;
4864		- }
4865		-
4866		- $value = get_post_meta($post_id, $key, true);
4867		-
4868		- // Only include non-empty string values
4869		- if (!empty($value) && is_string($value)) {
4870		- $result[$key] = $value;
4871		- } elseif (!empty($value) && is_array($value)) {
4872		- // Handle array values by joining them
4873		- $flat_value = $this->mxchat_flatten_meta_array($value);
4874		- if (!empty($flat_value)) {
4875		- $result[$key] = $flat_value;
4876		- }
4877		- }
4878		- }
4879		-
4880		- return $result;
4881		-}
4882		-
4883		-/**
4884		- * Flatten array meta values into a readable string
4885		- */
4886		-private function mxchat_flatten_meta_array($array, $depth = 0) {
4887		- if ($depth > 3) {
4888		- return ''; // Prevent infinite recursion
4889		- }
4890		-
4891		- $parts = array();
4892		-
4893		- foreach ($array as $key => $value) {
4894		- if (is_string($value) && !empty($value)) {
4895		- $parts[] = $value;
4896		- } elseif (is_array($value)) {
4897		- $nested = $this->mxchat_flatten_meta_array($value, $depth + 1);
4898		- if (!empty($nested)) {
4899		- $parts[] = $nested;
4900		- }
4901		- }
4902		- }
4903		-
4904		- return implode(', ', $parts);
4905		-}
4906		-
4907		-/**
4908		- * Format ACF field values for content extraction
4909		- */
4910		-public function mxchat_format_acf_field_value($value, $field_name = '', $post_id = 0) {
4911		- if (empty($value)) {
4912		- return '';
4913		- }
4914		-
4915		- // Handle WP_Post objects first (THIS IS THE KEY FIX)
4916		- if ($value instanceof WP_Post) {
4917		- return $value->post_title ?: '';
4918		- }
4919		-
4920		- // Handle other WP objects
4921		- if (is_object($value)) {
4922		- if (isset($value->post_title)) {
4923		- return $value->post_title;
4924		- } elseif (isset($value->display_name)) {
4925		- return $value->display_name;
4926		- } elseif (isset($value->name)) {
4927		- return $value->name;
4928		- } elseif (method_exists($value, '__toString')) {
4929		- try {
4930		- return (string) $value;
4931		- } catch (Exception $e) {
4932		- return '';
4933		- }
4934		- }
4935		- // For any other objects, return empty string
4936		- return '';
4937		- }
4938		-
4939		- // Handle different ACF field types
4940		- if (is_array($value)) {
4941		- // Check if it's an image/file field
4942		- if (isset($value['url'])) {
4943		- // Image field - return alt text, title, or caption
4944		- if (!empty($value['alt'])) {
4945		- return $value['alt'];
4946		- } elseif (!empty($value['title'])) {
4947		- return $value['title'];
4948		- } elseif (!empty($value['caption'])) {
4949		- return $value['caption'];
4950		- } else {
4951		- return ''; // Don't include just the URL
4952		- }
4953		- }
4954		-
4955		- // Check if it's a post object or relationship field
4956		- if (isset($value['post_title'])) {
4957		- return $value['post_title'];
4958		- }
4959		-
4960		- // Check if it's a user field
4961		- if (isset($value['display_name'])) {
4962		- return $value['display_name'];
4963		- }
4964		-
4965		- // Check if it's a taxonomy term
4966		- if (isset($value['name']) && isset($value['taxonomy'])) {
4967		- return $value['name'];
4968		- }
4969		-
4970		- // Check if it's a select field with label
4971		- if (isset($value['label'])) {
4972		- return $value['label'];
4973		- }
4974		-
4975		- // Check for repeater field or flexible content
4976		- if (is_numeric(key($value))) {
4977		- $sub_values = array();
4978		- foreach ($value as $sub_item) {
4979		- if (is_array($sub_item)) {
4980		- // For repeater/flexible content, extract text values
4981		- $sub_text = $this->mxchat_extract_text_from_acf_array($sub_item);
4982		- if (!empty($sub_text)) {
4983		- $sub_values[] = $sub_text;
4984		- }
4985		- } elseif ($sub_item instanceof WP_Post) {
4986		- // Handle WP_Post objects in arrays
4987		- $sub_values[] = $sub_item->post_title ?: '';
4988		- } else {
4989		- $sub_values[] = (string) $sub_item;
4990		- }
4991		- }
4992		- return implode(', ', array_filter($sub_values));
4993		- }
4994		-
4995		- // For other arrays, try to extract meaningful text
4996		- $text_values = array();
4997		- foreach ($value as $key => $val) {
4998		- if (is_string($val) && !empty(trim($val))) {
4999		- $text_values[] = trim($val);
5000		- } elseif ($val instanceof WP_Post) {
5001		- // Handle WP_Post objects in associative arrays
5002		- $text_values[] = $val->post_title ?: '';
5003		- } elseif (is_array($val) && isset($val['post_title'])) {
5004		- $text_values[] = $val['post_title'];
5005		- } elseif (is_array($val) && isset($val['name'])) {
5006		- $text_values[] = $val['name'];
5007		- }
5008		- }
5009		-
5010		- return implode(', ', array_filter($text_values));
5011		- }
5012		-
5013		- // Handle boolean values
5014		- if (is_bool($value)) {
5015		- return $value ? 'Yes' : 'No';
5016		- }
5017		-
5018		- // Handle numeric values
5019		- if (is_numeric($value)) {
5020		- return (string) $value;
5021		- }
5022		-
5023		- // Handle string values
5024		- if (is_string($value)) {
5025		- return trim($value);
5026		- }
5027		-
5028		- // For anything else that we can't handle, return empty string
5029		- // This prevents the "Object could not be converted to string" error
5030		- return '';
5031		-}
5032		-
5033		-/**
5034		- * Extract text from complex ACF array structures
5035		- */
5036		-private function mxchat_extract_text_from_acf_array($array) {
5037		- if (!is_array($array)) {
5038		- return '';
5039		- }
5040		-
5041		- $text_parts = array();
5042		-
5043		- foreach ($array as $key => $value) {
5044		- if (is_string($value) && !empty(trim($value))) {
5045		- // Skip keys that are likely to be IDs or technical values
5046		- if (!is_numeric($value) || strlen($value) > 10) {
5047		- $text_parts[] = trim($value);
5048		- }
5049		- } elseif ($value instanceof WP_Post) {
5050		- // Handle WP_Post objects
5051		- $text_parts[] = $value->post_title ?: '';
5052		- } elseif (is_array($value)) {
5053		- if (isset($value['post_title'])) {
5054		- $text_parts[] = $value['post_title'];
5055		- } elseif (isset($value['name'])) {
5056		- $text_parts[] = $value['name'];
5057		- } elseif (isset($value['label'])) {
5058		- $text_parts[] = $value['label'];
5059		- }
5060		- } elseif (is_object($value)) {
5061		- // Handle other objects safely
5062		- if (isset($value->post_title)) {
5063		- $text_parts[] = $value->post_title;
5064		- } elseif (isset($value->name)) {
5065		- $text_parts[] = $value->name;
5066		- } elseif (isset($value->display_name)) {
5067		- $text_parts[] = $value->display_name;
5068		- }
5069		- }
5070		- }
5071		-
5072		- return implode(', ', array_filter($text_parts));
5073		-}
5074		-
5075		-/**
5076		- * Handle ACF save - fires after ACF fields are saved
5077		- * This ensures ACF field data is available when syncing to knowledge base
5078		- */
5079		-public function mxchat_handle_acf_save($post_id) {
5080		- // Skip if not a valid post
5081		- if (!$post_id || $post_id === 'options') {
5082		- return;
5083		- }
5084		-
5085		- // Skip autosaves and revisions
	2953	+public function mxchat_handle_post_update($post_id, $post, $update) {
	2954	+ // Basic validation checks
5086	2955	if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5087	2956	return;
5088	2957	}
5089		-
5090		- $post = get_post($post_id);
5091		- if (!$post) {
5092		- return;
5093		- }
5094		-
5095		- $post_type = $post->post_type;
5096		-
5097		- // Check if sync is enabled for this post type
5098		- $should_sync = false;
5099		-
5100		- if ($post_type === 'post' && get_option('mxchat_auto_sync_posts') === '1') {
5101		- $should_sync = true;
5102		- } else if ($post_type === 'page' && get_option('mxchat_auto_sync_pages') === '1') {
5103		- $should_sync = true;
5104		- } else if ($post_type === 'product' && class_exists('WooCommerce')) {
5105		- // WooCommerce products - check if WooCommerce integration is enabled
5106		- $options = get_option('mxchat_options', array());
5107		- if (isset($options['enable_woocommerce_integration']) &&
5108		- ($options['enable_woocommerce_integration'] === '1' || $options['enable_woocommerce_integration'] === 'on')) {
5109		- $should_sync = true;
5110		- }
5111		- } else {
5112		- // Check custom post types
5113		- $option_name = 'mxchat_auto_sync_' . $post_type;
5114		- if (get_option($option_name) === '1') {
5115		- $should_sync = true;
5116		- }
5117		- }
5118		-
5119		- if (!$should_sync) {
5120		- return;
5121		- }
5122		-
5123		- // Only process published posts
	2958	+
	2959	+ // Only process published content
5124	2960	if ($post->post_status !== 'publish') {
5125	2961	return;
5126	2962	}
5127		-
5128		- // Check if this post has any ACF fields - if not, no need to re-sync
5129		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
5130		- if (empty($acf_fields)) {
5131		- return;
5132		- }
5133		-
5134		- // Use a transient to prevent duplicate processing (post_updated may have already run)
5135		- $transient_key = 'mxchat_acf_synced_' . $post_id;
5136		- if (get_transient($transient_key)) {
5137		- return;
5138		- }
5139		- set_transient($transient_key, true, 60); // Prevent re-processing for 60 seconds
5140		-
5141		- // Re-run the sync with ACF data now available
5142		- // We pass $update=true since this is effectively an update with ACF data
5143		- $this->mxchat_handle_post_update($post_id, $post, true);
5144		-}
5145		-
5146		-public function mxchat_handle_post_update($post_id, $post, $update) {
5147		- // Basic validation checks
5148		- if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5149		- return;
5150		- }
5151	2963
5152	2964	$post_type = $post->post_type;
5153	2965
5154	2966	// Check if sync is enabled for this post type
		@@ -5170,244 +2982,80 @@
5170	2982	if (!$should_sync) {
5171	2983	return;
5172	2984	}
5173	2985
5174		- // Check if we have stored the previous status and URL in our transients
5175		- $previous_status_key = 'mxchat_prev_status_' . $post_id;
5176		- $previous_status = get_transient($previous_status_key);
	2986	+ // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
	2987	+ $title = get_the_title($post_id);
	2988	+ $content = get_post_field('post_content', $post_id);
5177	2989
5178		- $previous_url_key = 'mxchat_prev_url_' . $post_id;
5179		- $previous_url = get_transient($previous_url_key);
	2990	+ // Apply WordPress content filters to get properly formatted content
	2991	+ $content = apply_filters('the_content', $content);
5180	2992
5181		- // If the post was previously published but is now not published, remove from knowledge base
5182		- if ($previous_status === 'publish' && $post->post_status !== 'publish') {
5183		- // Use the stored URL from when it was published, or fall back to current permalink
5184		- $source_url = $previous_url ?: get_permalink($post_id);
5185		-
5186		- if ($source_url) {
5187		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
5188		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	2993	+ // Strip tags but preserve structure
	2994	+ $content = wp_strip_all_tags($content);
	2995	+
	2996	+ // Combine title and content
	2997	+ $final_content = $title . "\n\n" . $content;
	2998	+
	2999	+ // For custom post types like job_listing, include additional fields
	3000	+ if ($post_type === 'job_listing') {
	3001	+ // Add job-specific meta if available
	3002	+ $job_location = get_post_meta($post_id, '_job_location', true);
	3003	+ if (!empty($job_location)) {
	3004	+ $final_content .= "\n\nLocation: " . $job_location;
5189	3005	}
5190		-
5191		- // Clean up the transients and exit early
5192		- delete_transient($previous_status_key);
5193		- delete_transient($previous_url_key);
5194		- return;
5195		- }
5196		-
5197		- // Slug/permalink rename while still published: delete the old vectors before upserting new ones.
5198		- // Without this, md5(old_url) vectors (base + chunks) would be orphaned under the stale URL.
5199		- if ($post->post_status === 'publish' && !empty($previous_url)) {
5200		- $current_url = get_permalink($post_id);
5201		- if ($current_url && $current_url !== $previous_url) {
5202		- MxChat_Utils::delete_chunks_for_url($previous_url, 'default');
5203		- }
5204		- }
5205		-
5206		- // Store the current status for next time (if this is an update)
5207		- if ($update) {
5208		- set_transient($previous_status_key, $post->post_status, DAY_IN_SECONDS);
5209		-
5210		- // If the post is currently published, also store its URL
5211		- if ($post->post_status === 'publish') {
5212		- $current_url = get_permalink($post_id);
5213		- set_transient($previous_url_key, $current_url, DAY_IN_SECONDS);
5214		- }
5215		- }
5216		-
5217		- // Only process currently published content for adding/updating
5218		- if ($post->post_status === 'publish') {
5219		- // Get the source URL
5220		- $source_url = get_permalink($post_id);
5221	3006
5222		- // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
5223		- $title = get_the_title($post_id);
5224		- $content = get_post_field('post_content', $post_id);
5225		- $excerpt = get_post_field('post_excerpt', $post_id);
5226		-
5227		- // Remove shortcode tags but preserve content inside them
5228		- $content = $this->strip_shortcode_tags_preserve_content($content);
5229		- $excerpt = $this->strip_shortcode_tags_preserve_content($excerpt);
5230		-
5231		- // Strip tags but preserve structure (don't use 'the_content' filter as it may re-add shortcodes)
5232		- $content = wp_strip_all_tags($content);
5233		-
5234		- // Combine title, short description (if exists), and content
5235		- $final_content = $title . "\n\n";
5236		-
5237		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
5238		- if (!empty($excerpt)) {
5239		- $final_content .= "Short Description: " . wp_strip_all_tags($excerpt) . "\n\n";
5240		- }
5241		-
5242		- $final_content .= $content;
5243		-
5244		- // For WooCommerce products, include pricing and product details
5245		- if ($post_type === 'product' && class_exists('WooCommerce')) {
5246		- $product = wc_get_product($post_id);
5247		-
5248		- if ($product) {
5249		- // Get pricing information
5250		- $regular_price = $product->get_regular_price();
5251		- $sale_price = $product->get_sale_price();
5252		- $price = $product->get_price();
5253		- $sku = $product->get_sku();
5254		-
5255		- // Get currency symbol
5256		- $currency_symbol = get_woocommerce_currency_symbol();
5257		-
5258		- // Add pricing information
5259		- $final_content .= "\n";
5260		- if (!empty($regular_price)) {
5261		- $final_content .= "Price: " . $currency_symbol . $regular_price . "\n";
5262		- } elseif (!empty($price)) {
5263		- $final_content .= "Price: " . $currency_symbol . $price . "\n";
5264		- }
5265		-
5266		- if (!empty($sale_price) && $sale_price !== $regular_price) {
5267		- $final_content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
5268		- }
5269		-
5270		- // Handle variable products - show price range
5271		- if ($product->is_type('variable')) {
5272		- $min_price = $product->get_variation_price('min');
5273		- $max_price = $product->get_variation_price('max');
5274		- if ($min_price !== $max_price) {
5275		- $final_content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
5276		- }
5277		- }
5278		-
5279		- if (!empty($sku)) {
5280		- $final_content .= "SKU: " . $sku . "\n";
5281		- }
5282		-
5283		- // Get product categories
5284		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
5285		- if (!empty($categories) && !is_wp_error($categories)) {
5286		- $final_content .= "Categories: " . implode(', ', $categories) . "\n";
5287		- }
	3007	+ // Get job type terms
	3008	+ $job_types = get_the_terms($post_id, 'job_listing_type');
	3009	+ if (!empty($job_types) && !is_wp_error($job_types)) {
	3010	+ $types = array();
	3011	+ foreach ($job_types as $type) {
	3012	+ $types[] = $type->name;
5288	3013	}
	3014	+ $final_content .= "\n\nJob Type: " . implode(', ', $types);
5289	3015	}
5290		-
5291		- // For custom post types like job_listing, include additional fields
5292		- if ($post_type === 'job_listing') {
5293		- // Add job-specific meta if available
5294		- $job_location = get_post_meta($post_id, '_job_location', true);
5295		- if (!empty($job_location)) {
5296		- $final_content .= "\n\nLocation: " . $job_location;
5297		- }
5298		-
5299		- // Get job type terms
5300		- $job_types = get_the_terms($post_id, 'job_listing_type');
5301		- if (!empty($job_types) && !is_wp_error($job_types)) {
5302		- $types = array();
5303		- foreach ($job_types as $type) {
5304		- $types[] = $type->name;
5305		- }
5306		- $final_content .= "\n\nJob Type: " . implode(', ', $types);
5307		- }
5308		-
5309		- // Get company name if available
5310		- $company_name = get_post_meta($post_id, '_company_name', true);
5311		- if (!empty($company_name)) {
5312		- $final_content .= "\n\nCompany: " . $company_name;
5313		- }
5314		- }
5315		-
5316		- // ADD ACF FIELDS SUPPORT (matches ajax_mxchat_process_selected_content behavior)
5317		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
5318		- if (!empty($acf_fields)) {
5319		- $acf_content_parts = array();
5320		-
5321		- foreach ($acf_fields as $field_name => $field_value) {
5322		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
5323		- if (!empty($formatted_value)) {
5324		- // Convert field name to readable label
5325		- $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name));
5326		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
5327		- }
5328		- }
5329		-
5330		- if (!empty($acf_content_parts)) {
5331		- $final_content .= "\n\n" . implode("\n", $acf_content_parts);
5332		- }
5333		- }
5334		-
5335		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
5336		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
5337		- if (!empty($custom_meta)) {
5338		- $meta_content_parts = array();
5339		-
5340		- foreach ($custom_meta as $meta_key => $meta_value) {
5341		- // Convert meta key to readable label
5342		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
5343		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
5344		- }
5345		-
5346		- if (!empty($meta_content_parts)) {
5347		- $final_content .= "\n\n" . implode("\n", $meta_content_parts);
5348		- }
5349		- }
5350		-
5351		- // Get API key with proper model detection
5352		- $options = get_option('mxchat_options');
5353		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
5354	3016
5355		- if (strpos($selected_model, 'voyage') === 0) {
5356		- $api_key = $options['voyage_api_key'] ?? '';
5357		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
5358		- $api_key = $options['gemini_api_key'] ?? '';
5359		- } else {
5360		- $api_key = $options['api_key'] ?? '';
	3017	+ // Get company name if available
	3018	+ $company_name = get_post_meta($post_id, '_company_name', true);
	3019	+ if (!empty($company_name)) {
	3020	+ $final_content .= "\n\nCompany: " . $company_name;
5361	3021	}
5362		-
5363		- if (empty($api_key)) {
5364		- return;
5365		- }
5366		-
5367		- // Use the centralized utility function for storage
5368		- $result = MxChat_Utils::submit_content_to_db(
5369		- $final_content,
5370		- $source_url,
5371		- $api_key,
5372		- md5($source_url) // Vector ID for Pinecone
5373		- );
5374		-
5375		- // After successful storage, apply role restriction based on tags
5376		- if (!is_wp_error($result)) {
5377		- $this->apply_role_restriction_to_post($post_id, $source_url);
5378		- }
5379	3022	}
5380	3023
5381		- // Clean up the stored previous status if not used above
5382		- if ($previous_status !== 'publish' || $post->post_status === 'publish') {
5383		- delete_transient($previous_status_key);
5384		- delete_transient($previous_url_key);
	3024	+ // Get the source URL
	3025	+ $source_url = get_permalink($post_id);
	3026	+
	3027	+ // Get API key with proper model detection
	3028	+ $options = get_option('mxchat_options');
	3029	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	3030	+
	3031	+ if (strpos($selected_model, 'voyage') === 0) {
	3032	+ $api_key = $options['voyage_api_key'] ?? '';
	3033	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	3034	+ $api_key = $options['gemini_api_key'] ?? '';
	3035	+ } else {
	3036	+ $api_key = $options['api_key'] ?? '';
5385	3037	}
5386		-}
5387		-
5388		-/**
5389		- * Store the post status and URL before update to detect status transitions
5390		- * This runs before the post is actually updated in the database
5391		- */
5392		-public function mxchat_store_pre_update_status($post_id, $data) {
5393		- // Get the current post from database (before update)
5394		- $current_post = get_post($post_id);
5395	3038
5396		- if ($current_post) {
5397		- // Store the current status temporarily
5398		- $status_key = 'mxchat_prev_status_' . $post_id;
5399		- set_transient($status_key, $current_post->post_status, HOUR_IN_SECONDS);
5400		-
5401		- // If the post is currently published, also store its URL
5402		- if ($current_post->post_status === 'publish') {
5403		- $url_key = 'mxchat_prev_url_' . $post_id;
5404		- $current_url = get_permalink($post_id);
5405		- set_transient($url_key, $current_url, HOUR_IN_SECONDS);
5406		- }
	3039	+ if (empty($api_key)) {
	3040	+ error_log('MxChat Auto-sync: No API key configured for embedding model');
	3041	+ return;
5407	3042	}
	3043	+
	3044	+ // Use the centralized utility function for storage
	3045	+ $result = MxChat_Utils::submit_content_to_db(
	3046	+ $final_content,
	3047	+ $source_url,
	3048	+ $api_key,
	3049	+ md5($source_url) // Vector ID for Pinecone
	3050	+ );
	3051	+
	3052	+ if (is_wp_error($result)) {
	3053	+ error_log('MxChat Auto-sync failed for post ' . $post_id . ': ' . $result->get_error_message());
	3054	+ }
5408	3055	}
5409	3056
	3057	+
5410	3058	public function mxchat_handle_post_delete($post_id) {
5411	3059	// Get post data before it's deleted
5412	3060	$post = get_post($post_id);
5413	3061
		@@ -5437,51 +3085,85 @@
5437	3085	if (!$should_sync) {
5438	3086	return;
5439	3087	}
5440	3088
5441		- // Resolve the pre-trash URL. wp_trash_post renames the slug with "__trashed" before firing
5442		- // this hook, so get_permalink() here would return the trashed URL and md5() would miss the
5443		- // real vector IDs stored under the original URL.
5444		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
	3089	+ // Get the URL before post is deleted
	3090	+ $source_url = get_permalink($post_id);
5445	3091	if (!$source_url) {
5446		- //error_log('MXChat: Failed to resolve source URL for post ' . $post_id);
	3092	+ error_log('MXChat: Failed to get permalink for post ' . $post_id);
5447	3093	return;
5448	3094	}
5449	3095
5450		- // Use chunk-aware deletion (handles both chunked and non-chunked content)
5451		- $delete_result = MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	3096	+ // Check if Pinecone is enabled
	3097	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	3098	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5452	3099
5453		- if (is_wp_error($delete_result)) {
5454		- //error_log('MXChat: Chunk-aware deletion failed for URL: ' . $source_url . ' - ' . $delete_result->get_error_message());
	3100	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3101	+ // Delete from Pinecone
	3102	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
	3103	+ } else {
	3104	+ // Delete from WordPress DB
	3105	+ global $wpdb;
	3106	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	3107	+
	3108	+ $result = $wpdb->delete(
	3109	+ $table_name,
	3110	+ array('source_url' => $source_url),
	3111	+ array('%s')
	3112	+ );
	3113	+
	3114	+ if ($result === false) {
	3115	+ error_log('MXChat: WordPress DB deletion failed for URL: ' . $source_url);
	3116	+ }
5455	3117	}
	3118	+}
5456	3119
5457		- delete_transient('mxchat_prev_url_' . $post_id);
5458		- delete_transient('mxchat_prev_status_' . $post_id);
5459		-}
5460	3120
5461		-/**
5462		- * Resolve the source URL for a post being trashed/deleted.
5463		- *
5464		- * Why: wp_trash_post appends "__trashed" to the slug before the wp_trash_post action fires, so
5465		- * get_permalink() returns a URL whose md5() won't match the vector IDs stored in Pinecone or
5466		- * the source_url rows in the WP DB. Prefer the URL captured by mxchat_store_pre_update_status
5467		- * (runs on pre_post_update, before the rename); fall back to stripping the __trashed suffix.
5468		- */
5469		-private function mxchat_resolve_pre_trash_url($post_id) {
5470		- $previous_url = get_transient('mxchat_prev_url_' . $post_id);
5471		- if (!empty($previous_url)) {
5472		- return $previous_url;
5473		- }
	3121	+ /**
	3122	+ * Deletes data from Pinecone using a source URL
	3123	+ */
	3124	+ public function mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options) {
	3125	+ $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
	3126	+ $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
5474	3127
5475		- $current = get_permalink($post_id);
5476		- if (!$current) {
5477		- return '';
5478		- }
5479		- return preg_replace('#__trashed(/?)$#', '$1', $current);
5480		-}
	3128	+ if (empty($host) || empty($api_key)) {
	3129	+ //error_log('MXChat: Pinecone deletion failed - missing configuration');
	3130	+ return false;
	3131	+ }
5481	3132
	3133	+ $api_endpoint = "https://{$host}/vectors/delete";
	3134	+ $vector_id = md5($source_url);
5482	3135
	3136	+ $request_body = array(
	3137	+ 'ids' => array($vector_id)
	3138	+ );
5483	3139
	3140	+ $response = wp_remote_post($api_endpoint, array(
	3141	+ 'headers' => array(
	3142	+ 'Api-Key' => $api_key,
	3143	+ 'accept' => 'application/json',
	3144	+ 'content-type' => 'application/json'
	3145	+ ),
	3146	+ 'body' => wp_json_encode($request_body),
	3147	+ 'timeout' => 30
	3148	+ ));
	3149	+
	3150	+ if (is_wp_error($response)) {
	3151	+ //error_log('MXChat: Pinecone deletion error - ' . $response->get_error_message());
	3152	+ return false;
	3153	+ }
	3154	+
	3155	+ $response_code = wp_remote_retrieve_response_code($response);
	3156	+ if ($response_code !== 200) {
	3157	+ //error_log('MXChat: Pinecone deletion failed with status ' . $response_code);
	3158	+ return false;
	3159	+ }
	3160	+
	3161	+ return true;
	3162	+ }
	3163	+
	3164	+
	3165	+
5484	3166	public function mxchat_handle_product_change($post_id, $post, $update) {
5485	3167	if ($post->post_type !== 'product') {
5486	3168	return;
5487	3169	}
		@@ -5503,12 +3185,11 @@
5503	3185	if (!isset($this->options['enable_woocommerce_integration']) ||
5504	3186	!in_array($this->options['enable_woocommerce_integration'], ['1', 'on'])) {
5505	3187	return;
5506	3188	}
5507		-
	3189	+
5508	3190	$source_url = get_permalink($product->get_id());
5509		- $product_id = $product->get_id();
5510		-
	3191	+
5511	3192	// Build product content
5512	3193	$title = $product->get_name();
5513	3194	$description = $product->get_description();
5514	3195	$short_description = $product->get_short_description();
		@@ -5513,92 +3194,35 @@
5513	3194	$description = $product->get_description();
5514	3195	$short_description = $product->get_short_description();
5515	3196	$regular_price = $product->get_regular_price();
5516	3197	$sale_price = $product->get_sale_price();
5517		- $price = $product->get_price();
5518	3198	$sku = $product->get_sku();
5519		-
5520		- // Get currency symbol
5521		- $currency_symbol = get_woocommerce_currency_symbol();
5522		-
	3199	+
5523	3200	// Format content consistently
5524	3201	$content = $title . "\n\n";
5525		-
5526		- if (!empty($short_description)) {
5527		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
5528		- }
5529		-
	3202	+
5530	3203	if (!empty($description)) {
5531	3204	$content .= wp_strip_all_tags($description) . "\n\n";
5532	3205	}
5533		-
5534		- // Add pricing information
5535		- if (!empty($regular_price)) {
5536		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
5537		- } elseif (!empty($price)) {
5538		- $content .= "Price: " . $currency_symbol . $price . "\n";
	3206	+
	3207	+ if (!empty($short_description)) {
	3208	+ $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
5539	3209	}
5540		-
5541		- if (!empty($sale_price) && $sale_price !== $regular_price) {
5542		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
	3210	+
	3211	+ $content .= "Price: $" . $regular_price . "\n";
	3212	+
	3213	+ if (!empty($sale_price)) {
	3214	+ $content .= "Sale Price: $" . $sale_price . "\n";
5543	3215	}
5544		-
5545		- // Handle variable products - show price range
5546		- if ($product->is_type('variable')) {
5547		- $min_price = $product->get_variation_price('min');
5548		- $max_price = $product->get_variation_price('max');
5549		- if ($min_price !== $max_price) {
5550		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
5551		- }
5552		- }
5553		-
	3216	+
5554	3217	if (!empty($sku)) {
5555	3218	$content .= "SKU: " . $sku . "\n";
5556	3219	}
5557		-
5558		- // Get product categories
5559		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
5560		- if (!empty($categories) && !is_wp_error($categories)) {
5561		- $content .= "Categories: " . implode(', ', $categories) . "\n";
5562		- }
5563		-
5564		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
5565		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
5566		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
5567		- foreach ($custom_tabs as $tab) {
5568		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
5569		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
5570		-
5571		- if (!empty($tab_title) && !empty($tab_content)) {
5572		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
5573		- }
5574		- }
5575		- }
5576		-
5577		- // Also check for reusable/saved tabs applied to this product
5578		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
5579		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
5580		- // Get the saved tabs option
5581		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
5582		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
5583		- foreach ($applied_saved_tabs as $saved_tab_id) {
5584		- if (isset($saved_tabs[$saved_tab_id])) {
5585		- $tab = $saved_tabs[$saved_tab_id];
5586		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
5587		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
5588		-
5589		- if (!empty($tab_title) && !empty($tab_content)) {
5590		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
5591		- }
5592		- }
5593		- }
5594		- }
5595		- }
5596		-
	3220	+
5597	3221	// Get API key with proper model detection
5598	3222	$options = get_option('mxchat_options');
5599	3223	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
5600		-
	3224	+
5601	3225	if (strpos($selected_model, 'voyage') === 0) {
5602	3226	$api_key = $options['voyage_api_key'] ?? '';
5603	3227	} elseif (strpos($selected_model, 'gemini-embedding') === 0) {
5604	3228	$api_key = $options['gemini_api_key'] ?? '';
		@@ -5604,14 +3228,14 @@
5604	3228	$api_key = $options['gemini_api_key'] ?? '';
5605	3229	} else {
5606	3230	$api_key = $options['api_key'] ?? '';
5607	3231	}
5608		-
	3232	+
5609	3233	if (empty($api_key)) {
5610		- //error_log('MxChat Auto-sync: No API key configured for embedding model');
	3234	+ error_log('MxChat Auto-sync: No API key configured for embedding model');
5611	3235	return;
5612	3236	}
5613		-
	3237	+
5614	3238	// Use the centralized utility function for storage
5615	3239	$result = MxChat_Utils::submit_content_to_db(
5616	3240	$content,
5617	3241	$source_url,
		@@ -5617,16 +3241,11 @@
5617	3241	$source_url,
5618	3242	$api_key,
5619	3243	md5($source_url) // Vector ID for Pinecone
5620	3244	);
5621		-
5622		- // After successful storage, apply role restriction based on tags
5623		- if (!is_wp_error($result)) {
5624		- $this->apply_role_restriction_to_post($product_id, $source_url);
5625		- }
5626		-
	3245	+
5627	3246	if (is_wp_error($result)) {
5628		- //error_log('MxChat WooCommerce sync failed for product ' . $product_id . ': ' . $result->get_error_message());
	3247	+ error_log('MxChat WooCommerce sync failed for product ' . $product->get_id() . ': ' . $result->get_error_message());
5629	3248	}
5630	3249	}
5631	3250
5632	3251	public function mxchat_handle_product_delete($post_id) {
		@@ -5633,1112 +3252,30 @@
5633	3252	if (get_post_type($post_id) !== 'product') {
5634	3253	return;
5635	3254	}
5636	3255
5637		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
5638		- if (!$source_url) {
5639		- return;
5640		- }
	3256	+ $source_url = get_permalink($post_id);
5641	3257
5642		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
5643		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
5644		-
5645		- delete_transient('mxchat_prev_url_' . $post_id);
5646		- delete_transient('mxchat_prev_status_' . $post_id);
5647		-}
5648		-
5649		-/**
5650		- * Handle individual Pinecone content deletion
5651		- */
5652		-public function mxchat_handle_pinecone_prompt_delete() {
5653		- // Check permissions
5654		- if (!current_user_can('manage_options')) {
5655		- wp_die(esc_html__('You do not have sufficient permissions.', 'mxchat'));
5656		- }
5657		-
5658		- // Verify nonce
5659		- if (!isset($_GET['_wpnonce']) || !wp_verify_nonce($_GET['_wpnonce'], 'mxchat_delete_pinecone_prompt_nonce')) {
5660		- wp_die(esc_html__('Security check failed.', 'mxchat'));
5661		- }
5662		-
5663		- $vector_id = isset($_GET['vector_id']) ? sanitize_text_field($_GET['vector_id']) : '';
5664		-
5665		- if (empty($vector_id)) {
5666		- set_transient('mxchat_admin_notice_error',
5667		- esc_html__('Invalid vector ID.', 'mxchat'),
5668		- 30
5669		- );
5670		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
5671		- exit;
5672		- }
5673		-
5674		- // Get Pinecone settings
	3258	+ // Check if Pinecone is enabled
5675	3259	$pinecone_options = get_option('mxchat_pinecone_addon_options', array());
5676	3260	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5677		-
5678		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
5679		- set_transient('mxchat_admin_notice_error',
5680		- esc_html__('Pinecone is not properly configured.', 'mxchat'),
5681		- 30
5682		- );
5683		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
5684		- exit;
5685		- }
5686		-
5687		- // Delete from Pinecone
5688		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
5689		- $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
5690		- $vector_id,
5691		- $pinecone_options['mxchat_pinecone_api_key'],
5692		- $pinecone_options['mxchat_pinecone_host']
5693		- );
5694		-
5695		- if ($result['success']) {
5696		- // No cache clearing needed since we removed caching
5697		- set_transient('mxchat_admin_notice_success',
5698		- esc_html__('Entry deleted successfully from Pinecone.', 'mxchat'),
5699		- 30
5700		- );
5701		- } else {
5702		- set_transient('mxchat_admin_notice_error',
5703		- esc_html__('Failed to delete entry: ', 'mxchat') . $result['message'],
5704		- 30
5705		- );
5706		- }
5707		-
5708		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
5709		- exit;
5710		-}
5711		-/**
5712		- * Handle individual Pinecone content deletion via AJAX
5713		- */
5714		-public function ajax_mxchat_delete_pinecone_prompt() {
5715		- // Verify nonce and permissions
5716		- if (!check_ajax_referer('mxchat_delete_pinecone_prompt_nonce', 'nonce', false)) {
5717		- wp_send_json_error('Invalid nonce');
5718		- exit;
5719		- }
5720		-
5721		- if (!current_user_can('manage_options')) {
5722		- wp_send_json_error('Unauthorized access');
5723		- exit;
5724		- }
5725		-
5726		- $vector_id = isset($_POST['vector_id']) ? sanitize_text_field($_POST['vector_id']) : '';
5727		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
5728		-
5729		- if (empty($vector_id)) {
5730		- wp_send_json_error('Missing vector ID');
5731		- exit;
5732		- }
5733		-
5734		- // Get bot-specific Pinecone settings
5735		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
5736		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
5737		-
5738		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5739		-
5740		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
5741		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
5742		- exit;
5743		- }
5744		-
5745		- // Delete from the correct Pinecone index
5746		- $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
5747		- $vector_id,
5748		- $pinecone_options['mxchat_pinecone_api_key'],
5749		- $pinecone_options['mxchat_pinecone_host']
5750		- );
5751		-
5752		- if ($result['success']) {
5753		- // No cache clearing needed since we removed caching
5754		- wp_send_json_success(array(
5755		- 'message' => 'Entry deleted successfully from Pinecone',
5756		- 'vector_id' => $vector_id,
5757		- 'bot_id' => $bot_id
5758		- ));
5759		- } else {
5760		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete from Pinecone: ' . $result['message'], array('vector_id' => $vector_id, 'bot_id' => $bot_id));
5761		- wp_send_json_error('Failed to delete from Pinecone: ' . $result['message']);
5762		- }
5763		-
5764		- exit;
5765		-}
5766	3261
5767		-/**
5768		- * Handle deletion of all chunks for a given source URL via AJAX
5769		- * Follows the same pattern as ajax_mxchat_delete_pinecone_prompt
5770		- */
5771		-public function ajax_mxchat_delete_chunks_by_url() {
5772		- // Verify nonce and permissions
5773		- if (!check_ajax_referer('mxchat_delete_chunks_nonce', 'nonce', false)) {
5774		- wp_send_json_error('Invalid nonce');
5775		- exit;
5776		- }
5777		-
5778		- if (!current_user_can('manage_options')) {
5779		- wp_send_json_error('Unauthorized access');
5780		- exit;
5781		- }
5782		-
5783		- $source_url = isset($_POST['source_url']) ? esc_url_raw($_POST['source_url']) : '';
5784		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
5785		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
5786		-
5787		- if (empty($source_url)) {
5788		- wp_send_json_error('Missing source URL');
5789		- exit;
5790		- }
5791		-
5792		- // Generate the base vector ID from the source URL (same as how chunks are created)
5793		- $base_vector_id = md5($source_url);
5794		-
5795		- if ($data_source === 'pinecone') {
5796		- // Get bot-specific Pinecone settings (same as working delete function)
5797		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
5798		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
5799		-
5800		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5801		-
5802		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
5803		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
5804		- exit;
5805		- }
5806		-
5807		- $api_key = $pinecone_options['mxchat_pinecone_api_key'];
5808		- $host = $pinecone_options['mxchat_pinecone_host'];
5809		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
5810		-
5811		- // Collect all vector IDs to delete
5812		- $vectors_to_delete = array();
5813		-
5814		- // Add the original single-vector ID (for non-chunked content)
5815		- $vectors_to_delete[] = $base_vector_id;
5816		-
5817		- // Use Pinecone list API to find all chunk vectors with this prefix
5818		- // NOTE: Pinecone List API is a GET request with query parameters, not POST
5819		- $prefix = $base_vector_id . '_chunk_';
5820		-
5821		- $query_params = array(
5822		- 'prefix' => $prefix,
5823		- 'limit' => 100
5824		- );
5825		-
5826		- if (!empty($namespace)) {
5827		- $query_params['namespace'] = $namespace;
5828		- }
5829		-
5830		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
5831		-
5832		- $list_response = wp_remote_get($list_url, array(
5833		- 'headers' => array(
5834		- 'Api-Key' => $api_key,
5835		- 'accept' => 'application/json'
5836		- ),
5837		- 'timeout' => 30
5838		- ));
5839		-
5840		- if (!is_wp_error($list_response)) {
5841		- $list_body_response = wp_remote_retrieve_body($list_response);
5842		- $list_data = json_decode($list_body_response, true);
5843		- if (!empty($list_data['vectors'])) {
5844		- foreach ($list_data['vectors'] as $vector) {
5845		- if (isset($vector['id'])) {
5846		- $vectors_to_delete[] = $vector['id'];
5847		- }
5848		- }
5849		- }
5850		- }
5851		-
5852		- if (empty($vectors_to_delete)) {
5853		- wp_send_json_success(array(
5854		- 'message' => 'No vectors found to delete',
5855		- 'source_url' => $source_url
5856		- ));
5857		- exit;
5858		- }
5859		-
5860		- // Delete all vectors using the same endpoint as the working function
5861		- $delete_url = "https://{$host}/vectors/delete";
5862		-
5863		- $delete_body = array(
5864		- 'ids' => $vectors_to_delete
5865		- );
5866		-
5867		- if (!empty($namespace)) {
5868		- $delete_body['namespace'] = $namespace;
5869		- }
5870		-
5871		- $delete_response = wp_remote_post($delete_url, array(
5872		- 'headers' => array(
5873		- 'Api-Key' => $api_key,
5874		- 'accept' => 'application/json',
5875		- 'content-type' => 'application/json'
5876		- ),
5877		- 'body' => wp_json_encode($delete_body),
5878		- 'timeout' => 30
5879		- ));
5880		-
5881		- if (is_wp_error($delete_response)) {
5882		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete chunks from Pinecone: ' . $delete_response->get_error_message(), array('source_url' => $source_url));
5883		- wp_send_json_error('Failed to delete from Pinecone: ' . $delete_response->get_error_message());
5884		- exit;
5885		- }
5886		-
5887		- $response_code = wp_remote_retrieve_response_code($delete_response);
5888		-
5889		- if ($response_code !== 200) {
5890		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Pinecone API error (HTTP ' . $response_code . ')', array('source_url' => $source_url));
5891		- wp_send_json_error('Pinecone API error (HTTP ' . $response_code . ')');
5892		- exit;
5893		- }
5894		-
5895		- wp_send_json_success(array(
5896		- 'message' => 'All chunks deleted successfully from Pinecone',
5897		- 'source_url' => $source_url,
5898		- 'deleted_count' => count($vectors_to_delete)
5899		- ));
5900		-
	3262	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3263	+ // Delete from Pinecone
	3264	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
5901	3265	} else {
5902		- // WordPress database deletion
	3266	+ // Delete from WordPress DB
5903	3267	global $wpdb;
5904	3268	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
5905		-
5906		- $result = $wpdb->delete(
5907		- $table_name,
5908		- array('source_url' => $source_url),
5909		- array('%s')
5910		- );
5911		-
5912		- if ($result === false) {
5913		- MxChat_Admin::mxchat_log_debug('database_error', 'Failed to delete from database: ' . $wpdb->last_error, array('source_url' => $source_url));
5914		- wp_send_json_error('Failed to delete from database: ' . $wpdb->last_error);
5915		- exit;
5916		- }
5917		-
5918		- wp_send_json_success(array(
5919		- 'message' => 'All chunks deleted successfully from database',
5920		- 'source_url' => $source_url,
5921		- 'deleted_count' => $result
5922		- ));
5923		- }
5924		-
5925		- exit;
5926		-}
5927		-
5928		-/**
5929		- * Handle individual WordPress database content deletion via AJAX
5930		- * Mirrors the Pinecone delete handler but for WordPress database entries
5931		- */
5932		-public function ajax_mxchat_delete_wordpress_prompt() {
5933		- // Verify nonce and permissions
5934		- if (!check_ajax_referer('mxchat_delete_wordpress_prompt_nonce', 'nonce', false)) {
5935		- wp_send_json_error('Invalid nonce');
5936		- exit;
5937		- }
5938		-
5939		- if (!current_user_can('manage_options')) {
5940		- wp_send_json_error('Unauthorized access');
5941		- exit;
5942		- }
5943		-
5944		- $entry_id = isset($_POST['entry_id']) ? intval($_POST['entry_id']) : 0;
5945		-
5946		- if (empty($entry_id)) {
5947		- wp_send_json_error('Missing entry ID');
5948		- exit;
5949		- }
5950		-
5951		- global $wpdb;
5952		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
5953		-
5954		- // Clear cache for this entry
5955		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
5956		-
5957		- // Delete from database
5958		- $result = $wpdb->delete(
5959		- $table_name,
5960		- array('id' => $entry_id),
5961		- array('%d')
5962		- );
5963		-
5964		- if ($result !== false) {
5965		- wp_send_json_success(array(
5966		- 'message' => 'Entry deleted successfully',
5967		- 'entry_id' => $entry_id
5968		- ));
5969		- } else {
5970		- wp_send_json_error('Failed to delete entry from database');
5971		- }
5972		-
5973		- exit;
5974		-}
5975		-
5976		-/**
5977		- * Handle bulk deletion of knowledge entries via AJAX
5978		- * Supports both Pinecone and WordPress database entries
5979		- */
5980		-public function ajax_mxchat_bulk_delete_knowledge() {
5981		- // Verify nonce and permissions
5982		- if (!check_ajax_referer('mxchat_bulk_delete_knowledge_nonce', 'nonce', false)) {
5983		- wp_send_json_error('Invalid nonce');
5984		- exit;
5985		- }
5986		-
5987		- if (!current_user_can('manage_options')) {
5988		- wp_send_json_error('Unauthorized access');
5989		- exit;
5990		- }
5991		-
5992		- $entries = isset($_POST['entries']) ? $_POST['entries'] : array();
5993		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
5994		-
5995		- if (empty($entries) || !is_array($entries)) {
5996		- wp_send_json_error('No entries provided');
5997		- exit;
5998		- }
5999		-
6000		- // Extend execution time — bulk Pinecone operations can take a while
6001		- if (function_exists('set_time_limit')) {
6002		- set_time_limit(120);
6003		- }
6004		-
6005		- $success_ids = array();
6006		- $failed_ids = array();
6007		- $errors = array();
6008		-
6009		- global $wpdb;
6010		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6011		-
6012		- // Get Pinecone manager for Pinecone deletions
6013		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6014		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6015		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6016		-
6017		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
6018		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
6019		-
6020		- // =============================================
6021		- // PHASE 1: Collect all Pinecone vector IDs
6022		- // and separate WordPress entries
6023		- // =============================================
6024		- $pinecone_entry_ids = array(); // entry IDs that are pinecone-sourced
6025		- $wordpress_entries = array(); // entries for WordPress DB deletion
6026		- $all_vector_ids = array(); // all pinecone vector IDs to delete in one batch
6027		-
6028		- foreach ($entries as $entry) {
6029		- $entry_id = sanitize_text_field($entry['id'] ?? '');
6030		- $source = sanitize_text_field($entry['source'] ?? 'wordpress');
6031		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
6032		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
6033		-
6034		- if (empty($entry_id)) {
6035		- continue;
6036		- }
6037		-
6038		- if ($source === 'pinecone') {
6039		- if (!$use_pinecone || empty($api_key)) {
6040		- $failed_ids[] = $entry_id;
6041		- $errors[] = "Pinecone not configured for entry: $entry_id";
6042		- continue;
6043		- }
6044		-
6045		- $pinecone_entry_ids[] = $entry_id;
6046		-
6047		- if ($is_group && !empty($source_url)) {
6048		- // Grouped/chunked entry: collect base ID + chunk IDs via List API
6049		- $base_vector_id = md5($source_url);
6050		- $all_vector_ids[] = $base_vector_id;
6051		-
6052		- $list_url = 'https://' . $host . '/vectors/list?prefix=' . urlencode($base_vector_id . '_chunk_') . '&limit=100';
6053		- $list_response = wp_remote_get($list_url, array(
6054		- 'headers' => array(
6055		- 'Api-Key' => $api_key,
6056		- 'accept' => 'application/json'
6057		- ),
6058		- 'timeout' => 30
6059		- ));
6060		-
6061		- if (!is_wp_error($list_response)) {
6062		- $list_body = json_decode(wp_remote_retrieve_body($list_response), true);
6063		- if (!empty($list_body['vectors']) && is_array($list_body['vectors'])) {
6064		- foreach ($list_body['vectors'] as $vector) {
6065		- if (isset($vector['id'])) {
6066		- $all_vector_ids[] = $vector['id'];
6067		- }
6068		- }
6069		- }
6070		- }
6071		- } else {
6072		- // Single entry: the entry_id IS the vector ID
6073		- $all_vector_ids[] = $entry_id;
6074		- }
6075		- } else {
6076		- $wordpress_entries[] = $entry;
6077		- }
6078		- }
6079		-
6080		- // =============================================
6081		- // PHASE 2: Single batch delete to Pinecone
6082		- // =============================================
6083		- if (!empty($all_vector_ids)) {
6084		- $all_vector_ids = array_values(array_unique($all_vector_ids));
6085		- $pinecone_success = true;
6086		- $batches = array_chunk($all_vector_ids, 100);
6087		-
6088		- foreach ($batches as $batch) {
6089		- $delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
6090		- 'headers' => array(
6091		- 'Api-Key' => $api_key,
6092		- 'accept' => 'application/json',
6093		- 'content-type' => 'application/json'
6094		- ),
6095		- 'body' => wp_json_encode(array('ids' => $batch)),
6096		- 'timeout' => 60
6097		- ));
6098		-
6099		- if (is_wp_error($delete_response)) {
6100		- $pinecone_success = false;
6101		- $errors[] = 'Pinecone batch deletion failed: ' . $delete_response->get_error_message();
6102		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed: ' . $delete_response->get_error_message());
6103		- } else {
6104		- $response_code = wp_remote_retrieve_response_code($delete_response);
6105		- if ($response_code !== 200) {
6106		- $pinecone_success = false;
6107		- $response_body = wp_remote_retrieve_body($delete_response);
6108		- $errors[] = "Pinecone API error (HTTP $response_code)";
6109		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed (HTTP ' . $response_code . ')', array('response' => substr($response_body, 0, 200)));
6110		- }
6111		- }
6112		- }
6113		-
6114		- // Mark all pinecone entries based on batch result
6115		- foreach ($pinecone_entry_ids as $eid) {
6116		- if ($pinecone_success) {
6117		- $success_ids[] = $eid;
6118		- } else {
6119		- $failed_ids[] = $eid;
6120		- }
6121		- }
6122		- }
6123		-
6124		- // =============================================
6125		- // PHASE 3: WordPress database deletions
6126		- // =============================================
6127		- foreach ($wordpress_entries as $entry) {
6128		- $entry_id = sanitize_text_field($entry['id'] ?? '');
6129		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
6130		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
6131		-
6132		- if (empty($entry_id)) {
6133		- continue;
6134		- }
6135		-
6136		- try {
6137		- if ($is_group && !empty($source_url)) {
6138		- $result = $wpdb->delete(
6139		- $table_name,
6140		- array('source_url' => $source_url),
6141		- array('%s')
6142		- );
6143		- } else {
6144		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
6145		- $result = $wpdb->delete(
6146		- $table_name,
6147		- array('id' => intval($entry_id)),
6148		- array('%d')
6149		- );
6150		- }
6151		-
6152		- if ($result !== false) {
6153		- $success_ids[] = $entry_id;
6154		- } else {
6155		- $failed_ids[] = $entry_id;
6156		- $errors[] = "Database error for entry: $entry_id";
6157		- }
6158		- } catch (Exception $e) {
6159		- $failed_ids[] = $entry_id;
6160		- $errors[] = $e->getMessage();
6161		- }
6162		- }
6163		-
6164		- wp_send_json_success(array(
6165		- 'success_ids' => $success_ids,
6166		- 'failed_ids' => $failed_ids,
6167		- 'errors' => $errors,
6168		- 'total_processed' => count($success_ids) + count($failed_ids)
6169		- ));
6170		-
6171		- exit;
6172		-}
6173		-
6174		-/**
6175		- * Get hierarchical roles for dropdown
6176		- */
6177		-public function mxchat_get_role_options() {
6178		- return array(
6179		- 'public' => __('Public (Everyone)', 'mxchat'),
6180		- 'logged_in' => __('Logged In Users', 'mxchat'),
6181		- 'subscriber' => __('Subscribers & Above', 'mxchat'),
6182		- 'contributor' => __('Contributors & Above', 'mxchat'),
6183		- 'author' => __('Authors & Above', 'mxchat'),
6184		- 'editor' => __('Editors & Above', 'mxchat'),
6185		- 'administrator' => __('Administrators Only', 'mxchat')
6186		- );
6187		-}
6188		-
6189		-/**
6190		- * Check if user has access to content based on role restriction
6191		- */
6192		-public function mxchat_user_has_content_access($role_restriction) {
6193		- // Public content is always accessible
6194		- if ($role_restriction === 'public' || empty($role_restriction)) {
6195		- return true;
6196		- }
6197		-
6198		- // Check if user is logged in for logged_in restriction
6199		- if ($role_restriction === 'logged_in') {
6200		- return is_user_logged_in();
6201		- }
6202		-
6203		- // If not logged in, no access to role-restricted content
6204		- if (!is_user_logged_in()) {
6205		- return false;
6206		- }
6207		-
6208		- $user = wp_get_current_user();
6209		- $user_roles = $user->roles;
6210		-
6211		- if (empty($user_roles)) {
6212		- return false;
6213		- }
6214		-
6215		- // Define role hierarchy (higher number = higher access)
6216		- $hierarchy = array(
6217		- 'subscriber' => 1,
6218		- 'contributor' => 2,
6219		- 'author' => 3,
6220		- 'editor' => 4,
6221		- 'administrator' => 5
6222		- );
6223		-
6224		- // Get required level
6225		- $required_level = isset($hierarchy[$role_restriction]) ? $hierarchy[$role_restriction] : 0;
6226		-
6227		- // Check if user has required level or higher
6228		- foreach ($user_roles as $user_role) {
6229		- $user_level = isset($hierarchy[$user_role]) ? $hierarchy[$user_role] : 0;
6230		- if ($user_level >= $required_level) {
6231		- return true;
6232		- }
6233		- }
6234		-
6235		- return false;
6236		-}
6237		-
6238		-/**
6239		- * Handle role restriction updates via AJAX
6240		- * Removed cache clearing call since we removed caching
6241		- */
6242		-public function ajax_mxchat_update_role_restriction() {
6243		- // Verify nonce and permissions
6244		- if (!check_ajax_referer('mxchat_update_role_nonce', 'nonce', false)) {
6245		- wp_send_json_error('Invalid nonce');
6246		- exit;
6247		- }
6248		-
6249		- if (!current_user_can('manage_options')) {
6250		- wp_send_json_error('Unauthorized access');
6251		- exit;
6252		- }
6253		-
6254		- $entry_id = isset($_POST['entry_id']) ? sanitize_text_field($_POST['entry_id']) : '';
6255		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
6256		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
6257		-
6258		- if (empty($entry_id)) {
6259		- wp_send_json_error('Invalid entry ID');
6260		- exit;
6261		- }
6262		-
6263		- // Get knowledge manager instance to validate role restriction
6264		- $knowledge_manager = MxChat_Knowledge_Manager::get_instance();
6265		- $valid_roles = array_keys($knowledge_manager->mxchat_get_role_options());
6266		- if (!in_array($role_restriction, $valid_roles)) {
6267		- wp_send_json_error('Invalid role restriction');
6268		- exit;
6269		- }
6270		-
6271		- global $wpdb;
6272		-
6273		- if ($data_source === 'pinecone') {
6274		- // Handle Pinecone role restriction (stored separately in WordPress table)
6275		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6276	3269
6277		- // Use REPLACE to insert or update the role restriction
6278		- $result = $wpdb->replace(
6279		- $roles_table,
6280		- array(
6281		- 'vector_id' => $entry_id,
6282		- 'role_restriction' => $role_restriction,
6283		- 'updated_at' => current_time('mysql')
6284		- ),
6285		- array('%s', '%s', '%s')
6286		- );
6287		-
6288		- // No cache clearing needed since we removed caching
6289		-
6290		- } else {
6291		- // Handle WordPress database role restriction (existing functionality)
6292		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6293		-
6294		- $result = $wpdb->update(
	3270	+ $wpdb->delete(
6295	3271	$table_name,
6296		- array('role_restriction' => $role_restriction),
6297		- array('id' => absint($entry_id)),
6298		- array('%s'),
6299		- array('%d')
6300		- );
6301		- }
6302		-
6303		- if ($result === false) {
6304		- wp_send_json_error('Database update failed: ' . $wpdb->last_error);
6305		- exit;
6306		- }
6307		-
6308		- wp_send_json_success(array(
6309		- 'message' => 'Role restriction updated successfully',
6310		- 'role_restriction' => $role_restriction,
6311		- 'data_source' => $data_source,
6312		- 'entry_id' => $entry_id
6313		- ));
6314		- exit;
6315		-}
6316		-
6317		-// ========================================
6318		-// ROLE-BASED CONTENT RESTRICTIONS - NEW FUNCTIONS
6319		-// Add these to your MxChat_Knowledge_Manager class
6320		-// ========================================
6321		-
6322		-/**
6323		- * Initialize role-based content hooks
6324		- * Add this call to your __construct() or mxchat_init_hooks() method
6325		- */
6326		-private function mxchat_init_role_hooks() {
6327		- // AJAX handlers for tag-role mappings
6328		- add_action('wp_ajax_mxchat_add_tag_role_mapping', array($this, 'ajax_add_tag_role_mapping'));
6329		- add_action('wp_ajax_mxchat_delete_tag_role_mapping', array($this, 'ajax_delete_tag_role_mapping'));
6330		- add_action('wp_ajax_mxchat_get_tag_role_mappings', array($this, 'ajax_get_tag_role_mappings'));
6331		- add_action('wp_ajax_mxchat_bulk_update_tag_roles', array($this, 'ajax_bulk_update_tag_roles'));
6332		-
6333		- // Hook to automatically update role restrictions when tags are added/removed
6334		- add_action('set_object_terms', array($this, 'handle_tag_change'), 10, 6);
6335		-
6336		- // Hook to apply role restrictions on auto-sync
6337		- add_action('mxchat_content_stored', array($this, 'apply_role_restriction_after_storage'), 10, 2);
6338		-}
6339		-
6340		-/**
6341		- * Add tag-role mapping via AJAX
6342		- */
6343		-public function ajax_add_tag_role_mapping() {
6344		- // Verify nonce and permissions
6345		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6346		-
6347		- if (!current_user_can('manage_options')) {
6348		- wp_send_json_error('Unauthorized access');
6349		- exit;
6350		- }
6351		-
6352		- $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
6353		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
6354		-
6355		- if (empty($tag_slug)) {
6356		- wp_send_json_error('Tag slug is required');
6357		- exit;
6358		- }
6359		-
6360		- // Validate role restriction
6361		- $valid_roles = array_keys($this->mxchat_get_role_options());
6362		- if (!in_array($role_restriction, $valid_roles)) {
6363		- wp_send_json_error('Invalid role restriction');
6364		- exit;
6365		- }
6366		-
6367		- // Check if tag exists in WordPress
6368		- $term = get_term_by('slug', $tag_slug, 'post_tag');
6369		- if (!$term) {
6370		- wp_send_json_error('Tag does not exist in WordPress');
6371		- exit;
6372		- }
6373		-
6374		- // Get existing mappings
6375		- $mappings = get_option('mxchat_tag_role_mappings', array());
6376		-
6377		- // Check if mapping already exists
6378		- if (isset($mappings[$tag_slug])) {
6379		- wp_send_json_error('Mapping for this tag already exists');
6380		- exit;
6381		- }
6382		-
6383		- // Add new mapping
6384		- $mappings[$tag_slug] = $role_restriction;
6385		- update_option('mxchat_tag_role_mappings', $mappings);
6386		-
6387		- wp_send_json_success(array(
6388		- 'message' => 'Tag-role mapping added successfully',
6389		- 'tag_slug' => $tag_slug,
6390		- 'role_restriction' => $role_restriction
6391		- ));
6392		- exit;
6393		-}
6394		-
6395		-/**
6396		- * Delete tag-role mapping via AJAX
6397		- */
6398		-public function ajax_delete_tag_role_mapping() {
6399		- // Verify nonce and permissions
6400		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6401		-
6402		- if (!current_user_can('manage_options')) {
6403		- wp_send_json_error('Unauthorized access');
6404		- exit;
6405		- }
6406		-
6407		- $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
6408		-
6409		- if (empty($tag_slug)) {
6410		- wp_send_json_error('Tag slug is required');
6411		- exit;
6412		- }
6413		-
6414		- // Get existing mappings
6415		- $mappings = get_option('mxchat_tag_role_mappings', array());
6416		-
6417		- // Check if mapping exists
6418		- if (!isset($mappings[$tag_slug])) {
6419		- wp_send_json_error('Mapping does not exist');
6420		- exit;
6421		- }
6422		-
6423		- // Remove mapping
6424		- unset($mappings[$tag_slug]);
6425		- update_option('mxchat_tag_role_mappings', $mappings);
6426		-
6427		- wp_send_json_success(array(
6428		- 'message' => 'Tag-role mapping deleted successfully',
6429		- 'tag_slug' => $tag_slug
6430		- ));
6431		- exit;
6432		-}
6433		-
6434		-/**
6435		- * Get all tag-role mappings via AJAX
6436		- */
6437		-public function ajax_get_tag_role_mappings() {
6438		- // Verify nonce and permissions
6439		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6440		-
6441		- if (!current_user_can('manage_options')) {
6442		- wp_send_json_error('Unauthorized access');
6443		- exit;
6444		- }
6445		-
6446		- // Get mappings
6447		- $mappings = get_option('mxchat_tag_role_mappings', array());
6448		- $role_options = $this->mxchat_get_role_options();
6449		-
6450		- $formatted_mappings = array();
6451		-
6452		- foreach ($mappings as $tag_slug => $role_restriction) {
6453		- // Get tag object
6454		- $term = get_term_by('slug', $tag_slug, 'post_tag');
6455		-
6456		- // Count posts with this tag
6457		- $post_count = 0;
6458		- if ($term) {
6459		- $post_count = $term->count;
6460		- }
6461		-
6462		- $formatted_mappings[] = array(
6463		- 'tag_slug' => $tag_slug,
6464		- 'role_restriction' => $role_restriction,
6465		- 'role_label' => $role_options[$role_restriction] ?? $role_restriction,
6466		- 'post_count' => $post_count
6467		- );
6468		- }
6469		-
6470		- wp_send_json_success(array(
6471		- 'mappings' => $formatted_mappings
6472		- ));
6473		- exit;
6474		-}
6475		-
6476		-/**
6477		- * Bulk update role restrictions for all existing content with mapped tags
6478		- */
6479		-public function ajax_bulk_update_tag_roles() {
6480		- // Verify nonce and permissions
6481		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6482		-
6483		- if (!current_user_can('manage_options')) {
6484		- wp_send_json_error('Unauthorized access');
6485		- exit;
6486		- }
6487		-
6488		- // Get mappings
6489		- $mappings = get_option('mxchat_tag_role_mappings', array());
6490		-
6491		- if (empty($mappings)) {
6492		- wp_send_json_error('No tag-role mappings found');
6493		- exit;
6494		- }
6495		-
6496		- global $wpdb;
6497		-
6498		- // Check if using Pinecone
6499		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6500		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6501		-
6502		- $updated_count = 0;
6503		- $details = array();
6504		-
6505		- foreach ($mappings as $tag_slug => $role_restriction) {
6506		- // Get all posts with this tag
6507		- $posts = get_posts(array(
6508		- 'tag' => $tag_slug,
6509		- 'post_type' => 'any',
6510		- 'posts_per_page' => -1,
6511		- 'fields' => 'ids',
6512		- 'post_status' => 'publish'
6513		- ));
6514		-
6515		- if (empty($posts)) {
6516		- continue;
6517		- }
6518		-
6519		- $tag_updated = 0;
6520		-
6521		- foreach ($posts as $post_id) {
6522		- $source_url = get_permalink($post_id);
6523		- if (!$source_url) {
6524		- continue;
6525		- }
6526		-
6527		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
6528		- // Update Pinecone role restriction
6529		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6530		- $vector_id = md5($source_url);
6531		-
6532		- $result = $wpdb->replace(
6533		- $roles_table,
6534		- array(
6535		- 'vector_id' => $vector_id,
6536		- 'role_restriction' => $role_restriction,
6537		- 'updated_at' => current_time('mysql')
6538		- ),
6539		- array('%s', '%s', '%s')
6540		- );
6541		- } else {
6542		- // Update WordPress DB
6543		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6544		-
6545		- $result = $wpdb->update(
6546		- $table_name,
6547		- array('role_restriction' => $role_restriction),
6548		- array('source_url' => $source_url),
6549		- array('%s'),
6550		- array('%s')
6551		- );
6552		- }
6553		-
6554		- if ($result !== false) {
6555		- $tag_updated++;
6556		- $updated_count++;
6557		- }
6558		- }
6559		-
6560		- if ($tag_updated > 0) {
6561		- $details[] = sprintf(
6562		- 'Tag "%s" (%s): %d posts updated',
6563		- $tag_slug,
6564		- $role_restriction,
6565		- $tag_updated
6566		- );
6567		- }
6568		- }
6569		-
6570		- wp_send_json_success(array(
6571		- 'message' => 'Bulk update completed',
6572		- 'updated_count' => $updated_count,
6573		- 'tags_processed' => count($mappings),
6574		- 'details' => $details
6575		- ));
6576		- exit;
6577		-}
6578		-
6579		-/**
6580		- * Handle tag changes on posts (when tags are added or removed)
6581		- */
6582		-public function handle_tag_change($object_id, $terms, $tt_ids, $taxonomy, $append, $old_tt_ids) {
6583		- // Only process post tags
6584		- if ($taxonomy !== 'post_tag') {
6585		- return;
6586		- }
6587		-
6588		- // Get tag-role mappings
6589		- $mappings = get_option('mxchat_tag_role_mappings', array());
6590		-
6591		- if (empty($mappings)) {
6592		- return;
6593		- }
6594		-
6595		- // Get the post's URL
6596		- $source_url = get_permalink($object_id);
6597		- if (!$source_url) {
6598		- return;
6599		- }
6600		-
6601		- // Determine the highest role restriction based on tags
6602		- $highest_role = 'public';
6603		- $role_hierarchy = array(
6604		- 'public' => 0,
6605		- 'logged_in' => 1,
6606		- 'subscriber' => 2,
6607		- 'contributor' => 3,
6608		- 'author' => 4,
6609		- 'editor' => 5,
6610		- 'administrator' => 6
6611		- );
6612		-
6613		- // Get all current tags for the post
6614		- $current_tags = wp_get_post_tags($object_id, array('fields' => 'slugs'));
6615		-
6616		- // Find the highest role restriction among the tags
6617		- foreach ($current_tags as $tag_slug) {
6618		- if (isset($mappings[$tag_slug])) {
6619		- $role = $mappings[$tag_slug];
6620		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
6621		- $highest_role = $role;
6622		- }
6623		- }
6624		- }
6625		-
6626		- // Update the role restriction in the database
6627		- global $wpdb;
6628		-
6629		- // Check if using Pinecone
6630		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6631		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6632		-
6633		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
6634		- // Update Pinecone role restriction
6635		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6636		- $vector_id = md5($source_url);
6637		-
6638		- $wpdb->replace(
6639		- $roles_table,
6640		- array(
6641		- 'vector_id' => $vector_id,
6642		- 'role_restriction' => $highest_role,
6643		- 'updated_at' => current_time('mysql')
6644		- ),
6645		- array('%s', '%s', '%s')
6646		- );
6647		- } else {
6648		- // Update WordPress DB
6649		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6650		-
6651		- $wpdb->update(
6652		- $table_name,
6653		- array('role_restriction' => $highest_role),
6654	3272	array('source_url' => $source_url),
6655		- array('%s'),
6656	3273	array('%s')
6657	3274	);
6658	3275	}
6659	3276	}
6660		-
6661		-/**
6662		- * Apply role restriction after content is stored (for auto-sync)
6663		- */
6664		-public function apply_role_restriction_after_storage($post_id, $source_url) {
6665		- // Get tag-role mappings
6666		- $mappings = get_option('mxchat_tag_role_mappings', array());
6667	3277
6668		- if (empty($mappings)) {
6669		- return;
6670		- }
6671		-
6672		- // Get all tags for the post
6673		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
6674		-
6675		- if (empty($post_tags)) {
6676		- return;
6677		- }
6678		-
6679		- // Determine the highest role restriction based on tags
6680		- $highest_role = 'public';
6681		- $role_hierarchy = array(
6682		- 'public' => 0,
6683		- 'logged_in' => 1,
6684		- 'subscriber' => 2,
6685		- 'contributor' => 3,
6686		- 'author' => 4,
6687		- 'editor' => 5,
6688		- 'administrator' => 6
6689		- );
6690		-
6691		- foreach ($post_tags as $tag_slug) {
6692		- if (isset($mappings[$tag_slug])) {
6693		- $role = $mappings[$tag_slug];
6694		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
6695		- $highest_role = $role;
6696		- }
6697		- }
6698		- }
6699		-
6700		- // If no restricted tags found, return (leave as public)
6701		- if ($highest_role === 'public') {
6702		- return;
6703		- }
6704		-
6705		- // Update the role restriction
6706		- global $wpdb;
6707		-
6708		- // Check if using Pinecone
6709		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6710		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6711		-
6712		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
6713		- // Update Pinecone role restriction
6714		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6715		- $vector_id = md5($source_url);
6716		-
6717		- $wpdb->replace(
6718		- $roles_table,
6719		- array(
6720		- 'vector_id' => $vector_id,
6721		- 'role_restriction' => $highest_role,
6722		- 'updated_at' => current_time('mysql')
6723		- ),
6724		- array('%s', '%s', '%s')
6725		- );
6726		- } else {
6727		- // Update WordPress DB
6728		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6729		-
6730		- $wpdb->update(
6731		- $table_name,
6732		- array('role_restriction' => $highest_role),
6733		- array('source_url' => $source_url),
6734		- array('%s'),
6735		- array('%s')
6736		- );
6737		- }
6738		-}
6739		-
6740		-
6741	3278	// ========================================
6742	3279	// HELPER METHODS
6743	3280	// ========================================
6744	3281
		@@ -6805,1074 +3342,8 @@
6805	3342	*/
6806	3343	private function mxchat_get_pinecone_manager() {
6807	3344	return MxChat_Pinecone_Manager::get_instance();
6808	3345	}
6809		-
6810		-
6811		- // ========================================
6812		-// DATABASE QUEUE TABLE MANAGEMENT
6813		-// ========================================
6814		-
6815		-/**
6816		- * Create queue table on plugin activation
6817		- * Call this from your plugin activation hook
6818		- */
6819		-public function mxchat_create_queue_table() {
6820		- global $wpdb;
6821		-
6822		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
6823		- $charset_collate = $wpdb->get_charset_collate();
6824		-
6825		- $sql = "CREATE TABLE IF NOT EXISTS $table_name (
6826		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
6827		- queue_id varchar(64) NOT NULL,
6828		- item_type varchar(20) NOT NULL,
6829		- item_data longtext NOT NULL,
6830		- status varchar(20) NOT NULL DEFAULT 'pending',
6831		- bot_id varchar(50) NOT NULL DEFAULT 'default',
6832		- priority int(11) NOT NULL DEFAULT 0,
6833		- attempts int(11) NOT NULL DEFAULT 0,
6834		- max_attempts int(11) NOT NULL DEFAULT 3,
6835		- error_message text DEFAULT NULL,
6836		- created_at datetime NOT NULL,
6837		- started_at datetime DEFAULT NULL,
6838		- completed_at datetime DEFAULT NULL,
6839		- PRIMARY KEY (id),
6840		- KEY queue_id (queue_id),
6841		- KEY status (status),
6842		- KEY item_type (item_type),
6843		- KEY priority (priority)
6844		- ) $charset_collate;";
6845		-
6846		- require_once(ABSPATH . 'wp-admin/includes/upgrade.php');
6847		- dbDelta($sql);
6848		-
6849		- // Also create a meta table for queue metadata
6850		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
6851		-
6852		- $meta_sql = "CREATE TABLE IF NOT EXISTS $meta_table (
6853		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
6854		- queue_id varchar(64) NOT NULL,
6855		- meta_key varchar(255) NOT NULL,
6856		- meta_value longtext,
6857		- PRIMARY KEY (id),
6858		- KEY queue_id (queue_id),
6859		- KEY meta_key (meta_key)
6860		- ) $charset_collate;";
6861		-
6862		- dbDelta($meta_sql);
6863		-}
6864		-
6865		-/**
6866		- * Add items to the processing queue
6867		- *
6868		- * @param string $queue_id Unique identifier for this queue batch
6869		- * @param string $item_type Type of item (url, pdf_page)
6870		- * @param array $items Array of items to queue
6871		- * @param string $bot_id Bot ID for processing
6872		- * @return int Number of items queued
6873		- */
6874		-private function mxchat_add_to_queue($queue_id, $item_type, $items, $bot_id = 'default') {
6875		- global $wpdb;
6876		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
6877		-
6878		- $queued_count = 0;
6879		- $priority = 0;
6880		-
6881		- foreach ($items as $item) {
6882		- $result = $wpdb->insert(
6883		- $table_name,
6884		- array(
6885		- 'queue_id' => $queue_id,
6886		- 'item_type' => $item_type,
6887		- 'item_data' => wp_json_encode($item),
6888		- 'status' => 'pending',
6889		- 'bot_id' => $bot_id,
6890		- 'priority' => $priority,
6891		- 'attempts' => 0,
6892		- 'max_attempts' => 3,
6893		- 'created_at' => current_time('mysql')
6894		- ),
6895		- array('%s', '%s', '%s', '%s', '%s', '%d', '%d', '%d', '%s')
6896		- );
6897		-
6898		- if ($result) {
6899		- $queued_count++;
6900		- }
6901		-
6902		- $priority++; // Process in order
6903		- }
6904		-
6905		- return $queued_count;
6906		-}
6907		-
6908		-/**
6909		- * Store queue metadata (total counts, source URL, etc.)
6910		- */
6911		-private function mxchat_set_queue_meta($queue_id, $meta_key, $meta_value) {
6912		- global $wpdb;
6913		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
6914		-
6915		- // Check if meta exists
6916		- $existing = $wpdb->get_var($wpdb->prepare(
6917		- "SELECT id FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
6918		- $queue_id,
6919		- $meta_key
6920		- ));
6921		-
6922		- if ($existing) {
6923		- // Update
6924		- $wpdb->update(
6925		- $meta_table,
6926		- array('meta_value' => maybe_serialize($meta_value)),
6927		- array('queue_id' => $queue_id, 'meta_key' => $meta_key),
6928		- array('%s'),
6929		- array('%s', '%s')
6930		- );
6931		- } else {
6932		- // Insert
6933		- $wpdb->insert(
6934		- $meta_table,
6935		- array(
6936		- 'queue_id' => $queue_id,
6937		- 'meta_key' => $meta_key,
6938		- 'meta_value' => maybe_serialize($meta_value)
6939		- ),
6940		- array('%s', '%s', '%s')
6941		- );
6942		- }
6943		-}
6944		-
6945		-/**
6946		- * Get queue metadata
6947		- */
6948		-private function mxchat_get_queue_meta($queue_id, $meta_key) {
6949		- global $wpdb;
6950		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
6951		-
6952		- $value = $wpdb->get_var($wpdb->prepare(
6953		- "SELECT meta_value FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
6954		- $queue_id,
6955		- $meta_key
6956		- ));
6957		-
6958		- return maybe_unserialize($value);
6959		-}
6960		-
6961		-// ========================================
6962		-// AJAX QUEUE PROCESSING HANDLERS
6963		-// ========================================
6964		-
6965		-/**
6966		- * AJAX: Get next item from queue to process
6967		- */
6968		-public function ajax_mxchat_get_next_queue_item() {
6969		- // Verify nonce and permissions
6970		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
6971		-
6972		- if (!current_user_can('manage_options')) {
6973		- wp_send_json_error('Unauthorized access');
6974		- }
6975		-
6976		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
6977		-
6978		- if (empty($queue_id)) {
6979		- wp_send_json_error('Missing queue ID');
6980		- }
6981		-
6982		- global $wpdb;
6983		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
6984		-
6985		- // Get next pending item with retry logic for failed items
6986		- $next_item = $wpdb->get_row($wpdb->prepare(
6987		- "SELECT * FROM $table_name
6988		- WHERE queue_id = %s
6989		- AND status IN ('pending', 'failed')
6990		- AND attempts < max_attempts
6991		- ORDER BY priority ASC, id ASC
6992		- LIMIT 1",
6993		- $queue_id
6994		- ));
6995		-
6996		- if (!$next_item) {
6997		- // No more items - queue complete
6998		- wp_send_json_success(array(
6999		- 'complete' => true,
7000		- 'message' => 'Queue processing complete'
7001		- ));
7002		- }
7003		-
7004		- // Mark item as processing
7005		- $wpdb->update(
7006		- $table_name,
7007		- array(
7008		- 'status' => 'processing',
7009		- 'started_at' => current_time('mysql'),
7010		- 'attempts' => $next_item->attempts + 1
7011		- ),
7012		- array('id' => $next_item->id),
7013		- array('%s', '%s', '%d'),
7014		- array('%d')
7015		- );
7016		-
7017		- wp_send_json_success(array(
7018		- 'complete' => false,
7019		- 'item' => array(
7020		- 'id' => $next_item->id,
7021		- 'type' => $next_item->item_type,
7022		- 'data' => json_decode($next_item->item_data, true),
7023		- 'bot_id' => $next_item->bot_id,
7024		- 'attempt' => $next_item->attempts + 1
7025		- )
7026		- ));
7027		-}
7028		-
7029		-/**
7030		- * AJAX: Process a single queue item
7031		- */
7032		-public function ajax_mxchat_process_queue_item() {
7033		- // Verify nonce and permissions
7034		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7035		-
7036		- if (!current_user_can('manage_options')) {
7037		- wp_send_json_error('Unauthorized access');
7038		- }
7039		-
7040		- $item_id = isset($_POST['item_id']) ? absint($_POST['item_id']) : 0;
7041		- $item_type = isset($_POST['item_type']) ? sanitize_text_field($_POST['item_type']) : '';
7042		- $item_data = isset($_POST['item_data']) ? $_POST['item_data'] : array();
7043		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
7044		-
7045		- if (empty($item_id) || empty($item_type)) {
7046		- wp_send_json_error('Missing item data');
7047		- }
7048		-
7049		- global $wpdb;
7050		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7051		-
7052		- // Process based on item type
7053		- try {
7054		- set_time_limit(60); // Give processing 60 seconds
7055		-
7056		- $result = false;
7057		- $error_message = '';
7058		-
7059		- // Read item directly from DB to get queue_id and preserve special chars in item_data
7060		- // (POST round-trip through JS mangles characters like apostrophes in URLs)
7061		- $db_item = $wpdb->get_row($wpdb->prepare(
7062		- "SELECT queue_id, item_data FROM $table_name WHERE id = %d",
7063		- $item_id
7064		- ));
7065		- $item_queue_id = $db_item ? $db_item->queue_id : '';
7066		- if ($db_item && !empty($db_item->item_data)) {
7067		- $db_data = json_decode($db_item->item_data, true);
7068		- if (is_array($db_data)) {
7069		- $item_data = $db_data;
7070		- }
7071		- }
7072		-
7073		- switch ($item_type) {
7074		- case 'url':
7075		- $result = $this->mxchat_process_queue_url($item_data, $bot_id, $item_queue_id);
7076		- break;
7077		-
7078		- case 'pdf_page':
7079		- $result = $this->mxchat_process_queue_pdf_page($item_data, $bot_id);
7080		- break;
7081		-
7082		- default:
7083		- throw new Exception('Unknown item type: ' . $item_type);
7084		- }
7085		-
7086		- if (is_wp_error($result)) {
7087		- $error_code = $result->get_error_code();
7088		- // Content errors (empty page, sanitization) are permanent — retrying won't help
7089		- $permanent_codes = array('empty_page', 'empty_after_sanitization', 'no_api_key', 'page_not_found');
7090		- if (in_array($error_code, $permanent_codes)) {
7091		- // Mark as permanently failed — set attempts = max_attempts so it won't be retried
7092		- $current_item = $wpdb->get_row($wpdb->prepare(
7093		- "SELECT max_attempts FROM $table_name WHERE id = %d", $item_id
7094		- ));
7095		- $wpdb->update(
7096		- $table_name,
7097		- array(
7098		- 'status' => 'failed',
7099		- 'error_message' => $result->get_error_message(),
7100		- 'attempts' => $current_item ? $current_item->max_attempts : 3
7101		- ),
7102		- array('id' => $item_id),
7103		- array('%s', '%s', '%d'),
7104		- array('%d')
7105		- );
7106		- wp_send_json_error(array(
7107		- 'message' => $result->get_error_message(),
7108		- 'permanent_failure' => true,
7109		- 'item_id' => $item_id
7110		- ));
7111		- return;
7112		- }
7113		- throw new Exception($result->get_error_message());
7114		- }
7115		-
7116		- if ($result === false) {
7117		- throw new Exception('Processing returned false - item may be empty or invalid');
7118		- }
7119		-
7120		- // Mark as completed
7121		- $wpdb->update(
7122		- $table_name,
7123		- array(
7124		- 'status' => 'completed',
7125		- 'completed_at' => current_time('mysql'),
7126		- 'error_message' => null
7127		- ),
7128		- array('id' => $item_id),
7129		- array('%s', '%s', '%s'),
7130		- array('%d')
7131		- );
7132		-
7133		- wp_send_json_success(array(
7134		- 'processed' => true,
7135		- 'item_id' => $item_id,
7136		- 'message' => 'Item processed successfully'
7137		- ));
7138		-
7139		- } catch (Exception $e) {
7140		- $error_message = $e->getMessage();
7141		-
7142		- // Get current attempt count
7143		- $item = $wpdb->get_row($wpdb->prepare(
7144		- "SELECT attempts, max_attempts FROM $table_name WHERE id = %d",
7145		- $item_id
7146		- ));
7147		-
7148		- // Check if we've exhausted retries
7149		- if ($item && $item->attempts >= $item->max_attempts) {
7150		- // Permanently failed
7151		- $wpdb->update(
7152		- $table_name,
7153		- array(
7154		- 'status' => 'failed',
7155		- 'error_message' => $error_message
7156		- ),
7157		- array('id' => $item_id),
7158		- array('%s', '%s'),
7159		- array('%d')
7160		- );
7161		-
7162		- wp_send_json_error(array(
7163		- 'message' => 'Item failed after maximum attempts: ' . $error_message,
7164		- 'permanent_failure' => true,
7165		- 'item_id' => $item_id
7166		- ));
7167		- } else {
7168		- // Mark for retry
7169		- $wpdb->update(
7170		- $table_name,
7171		- array(
7172		- 'status' => 'failed',
7173		- 'error_message' => $error_message
7174		- ),
7175		- array('id' => $item_id),
7176		- array('%s', '%s'),
7177		- array('%d')
7178		- );
7179		-
7180		- wp_send_json_error(array(
7181		- 'message' => 'Item processing failed, will retry: ' . $error_message,
7182		- 'can_retry' => true,
7183		- 'item_id' => $item_id,
7184		- 'attempts' => $item ? $item->attempts : 0
7185		- ));
7186		- }
7187		- }
7188		-}
7189		-
7190		-/**
7191		- * Process a URL from the queue
7192		- */
7193		-private function mxchat_process_queue_url($item_data, $bot_id = 'default', $queue_id = '') {
7194		- $url = isset($item_data['url']) ? $item_data['url'] : '';
7195		-
7196		- if (empty($url)) {
7197		- return new WP_Error('invalid_url', 'URL is empty');
7198		- }
7199		-
7200		- // Get bot-specific API key early (needed for both paths)
7201		- $bot_options = $this->get_bot_options($bot_id);
7202		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
7203		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
7204		-
7205		- if (strpos($selected_model, 'voyage') === 0) {
7206		- $api_key = $options['voyage_api_key'] ?? '';
7207		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
7208		- $api_key = $options['gemini_api_key'] ?? '';
7209		- } else {
7210		- $api_key = $options['api_key'] ?? '';
7211		- }
7212		-
7213		- if (empty($api_key)) {
7214		- return new WP_Error('no_api_key', 'API key not configured for bot: ' . $bot_id);
7215		- }
7216		-
7217		- // Check if this is a WooCommerce product URL and WooCommerce is active
7218		- $is_product_url = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
7219		- $content_type = $is_product_url ? 'product' : 'url';
7220		-
7221		- // Try to get WooCommerce product data if it's a product URL
7222		- if ($is_product_url && class_exists('WooCommerce')) {
7223		- $product_content = $this->mxchat_extract_woocommerce_product_content($url);
7224		-
7225		- if (!empty($product_content)) {
7226		- // Successfully extracted WooCommerce product data with pricing
7227		- $result = MxChat_Utils::submit_content_to_db(
7228		- $product_content,
7229		- $url,
7230		- $api_key,
7231		- null,
7232		- $bot_id,
7233		- 'product'
7234		- );
7235		- return $result;
7236		- }
7237		- // If WooCommerce extraction failed, fall through to HTML extraction
7238		- }
7239		-
7240		- // Fetch URL content (fallback for non-products or when WooCommerce extraction fails)
7241		- $is_likely_pdf = (strtolower(pathinfo(parse_url($url, PHP_URL_PATH) ?: '', PATHINFO_EXTENSION)) === 'pdf');
7242		- $response = wp_remote_get($url, array(
7243		- 'timeout' => $is_likely_pdf ? 120 : 30,
7244		- 'redirection' => 5,
7245		- 'user-agent' => 'MxChat/1.0'
7246		- ));
7247		-
7248		- if (is_wp_error($response)) {
7249		- return $response;
7250		- }
7251		-
7252		- $response_code = wp_remote_retrieve_response_code($response);
7253		- if ($response_code !== 200) {
7254		- return new WP_Error('http_error', 'HTTP ' . $response_code . ' error for: ' . $url);
7255		- }
7256		-
7257		- // Check if URL is a PDF — expand into per-page queue items using the standard PDF pipeline
7258		- if ($this->mxchat_is_pdf_url($url, $response)) {
7259		- return $this->mxchat_expand_pdf_to_queue($url, $response, $bot_id, $queue_id);
7260		- }
7261		-
7262		- $html = wp_remote_retrieve_body($response);
7263		-
7264		- if (empty($html)) {
7265		- return new WP_Error('empty_response', 'Empty response body');
7266		- }
7267		-
7268		- // Extract and sanitize content
7269		- $content = $this->mxchat_extract_main_content($html);
7270		- $sanitized = $this->mxchat_sanitize_content_for_api($content);
7271		-
7272		- if (empty($sanitized)) {
7273		- // Not an error - just no content found (maybe a redirect or empty page)
7274		- return false;
7275		- }
7276		-
7277		- // Submit to database with content_type
7278		- $result = MxChat_Utils::submit_content_to_db(
7279		- $sanitized,
7280		- $url,
7281		- $api_key,
7282		- null,
7283		- $bot_id,
7284		- $content_type
7285		- );
7286		-
7287		- return $result;
7288		-}
7289		-
7290		-/**
7291		- * Expand a PDF URL into per-page queue items using the standard PDF pipeline.
7292		- * Called when a sitemap URL turns out to be a PDF — downloads, parses page count,
7293		- * and adds pdf_page items to the same queue so they process with full progress tracking.
7294		- */
7295		-private function mxchat_expand_pdf_to_queue($pdf_url, $response, $bot_id = 'default', $queue_id = '') {
7296		- set_time_limit(120); // PDFs need extra time for download + parsing
7297		-
7298		- $upload_dir = wp_upload_dir();
7299		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
7300		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
7301		-
7302		- $response_body = wp_remote_retrieve_body($response);
7303		- if (empty($response_body)) {
7304		- return new WP_Error('empty_pdf', 'Empty PDF response for: ' . $pdf_url);
7305		- }
7306		-
7307		- if (!wp_mkdir_p(dirname($pdf_path))) {
7308		- return new WP_Error('dir_error', 'Failed to create upload directory');
7309		- }
7310		-
7311		- file_put_contents($pdf_path, $response_body);
7312		-
7313		- if (!file_exists($pdf_path)) {
7314		- return new WP_Error('save_error', 'Failed to save PDF file');
7315		- }
7316		-
7317		- try {
7318		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
7319		-
7320		- if ($total_pages === false || $total_pages < 1) {
7321		- wp_delete_file($pdf_path);
7322		- return new WP_Error('no_pages', 'PDF has no pages: ' . $pdf_url);
7323		- }
7324		-
7325		- // Build per-page items identical to mxchat_handle_pdf_for_knowledge_base
7326		- $pages = array();
7327		- for ($i = 1; $i <= $total_pages; $i++) {
7328		- $pages[] = array(
7329		- 'pdf_path' => $pdf_path,
7330		- 'pdf_url' => $pdf_url,
7331		- 'page_number' => $i,
7332		- 'total_pages' => $total_pages
7333		- );
7334		- }
7335		-
7336		- // Add pdf_page items to the SAME queue so the JS picks them up automatically
7337		- if (!empty($queue_id)) {
7338		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
7339		- } else {
7340		- // Fallback: create a new PDF queue (shouldn't happen in sitemap flow)
7341		- $new_queue_id = 'pdf_' . md5($pdf_url . time());
7342		- $queued_count = $this->mxchat_add_to_queue($new_queue_id, 'pdf_page', $pages, $bot_id);
7343		- $this->mxchat_set_queue_meta($new_queue_id, 'source_url', $pdf_url);
7344		- $this->mxchat_set_queue_meta($new_queue_id, 'queue_type', 'pdf');
7345		- $this->mxchat_set_queue_meta($new_queue_id, 'total_items', $total_pages);
7346		- $this->mxchat_set_queue_meta($new_queue_id, 'bot_id', $bot_id);
7347		- $this->mxchat_set_queue_meta($new_queue_id, 'pdf_path', $pdf_path);
7348		- $this->mxchat_set_queue_meta($new_queue_id, 'created_at', current_time('mysql'));
7349		- }
7350		-
7351		- if ($queued_count === 0) {
7352		- wp_delete_file($pdf_path);
7353		- return new WP_Error('queue_error', 'Failed to add PDF pages to queue');
7354		- }
7355		-
7356		- // Return true so the original URL item is marked complete
7357		- // The new pdf_page items will be processed in subsequent batches
7358		- return true;
7359		-
7360		- } catch (Exception $e) {
7361		- if (file_exists($pdf_path)) {
7362		- wp_delete_file($pdf_path);
7363		- }
7364		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
7365		- }
7366		-}
7367		-
7368		-/**
7369		- * Legacy: Process a PDF URL inline during sitemap queue processing.
7370		- * @deprecated Use mxchat_expand_pdf_to_queue instead — kept for reference only.
7371		- */
7372		-private function mxchat_process_pdf_url_inline($pdf_url, $response, $api_key, $bot_id = 'default') {
7373		- set_time_limit(120); // PDFs need more time — downloading + parsing all pages
7374		-
7375		- $upload_dir = wp_upload_dir();
7376		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
7377		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
7378		-
7379		- $response_body = wp_remote_retrieve_body($response);
7380		- if (empty($response_body)) {
7381		- return new WP_Error('empty_pdf', 'Empty PDF response');
7382		- }
7383		-
7384		- if (!wp_mkdir_p(dirname($pdf_path))) {
7385		- return new WP_Error('dir_error', 'Failed to create upload directory');
7386		- }
7387		-
7388		- file_put_contents($pdf_path, $response_body);
7389		-
7390		- if (!file_exists($pdf_path)) {
7391		- return new WP_Error('save_error', 'Failed to save PDF file');
7392		- }
7393		-
7394		- try {
7395		- mxchat_load_pdf_parser();
7396		- $parser = new \Smalot\PdfParser\Parser();
7397		- $pdf = $parser->parseFile($pdf_path);
7398		- $pages = $pdf->getPages();
7399		- $total_pages = count($pages);
7400		-
7401		- if ($total_pages < 1) {
7402		- wp_delete_file($pdf_path);
7403		- return new WP_Error('no_pages', 'PDF has no pages');
7404		- }
7405		-
7406		- $processed = 0;
7407		- $skipped_pages = array();
7408		-
7409		- for ($i = 0; $i < $total_pages; $i++) {
7410		- $page_num = $i + 1;
7411		- $text = $pages[$i]->getText();
7412		- if (empty($text)) {
7413		- $skipped_pages[] = 'Page ' . $page_num . ': No text could be extracted — page may contain only images, links, or non-standard encoding';
7414		- continue;
7415		- }
7416		-
7417		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
7418		- if (empty($sanitized)) {
7419		- $skipped_pages[] = 'Page ' . $page_num . ': Text was extracted but contained only special characters, control codes, or unsupported content';
7420		- continue;
7421		- }
7422		-
7423		- $metadata = array(
7424		- 'document_type' => 'pdf',
7425		- 'total_pages' => $total_pages,
7426		- 'current_page' => $page_num,
7427		- 'source_url' => $pdf_url,
7428		- );
7429		-
7430		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
7431		- $page_url = esc_url($pdf_url . '#page=' . $page_num);
7432		-
7433		- MxChat_Utils::submit_content_to_db(
7434		- $content_with_metadata,
7435		- $page_url,
7436		- $api_key,
7437		- null,
7438		- $bot_id,
7439		- 'pdf'
7440		- );
7441		-
7442		- $processed++;
7443		- }
7444		-
7445		- // Clean up the temp PDF file
7446		- wp_delete_file($pdf_path);
7447		-
7448		- if (!empty($skipped_pages)) {
7449		- error_log('MxChat PDF: Skipped ' . count($skipped_pages) . ' of ' . $total_pages . ' pages: ' . implode('; ', $skipped_pages));
7450		- }
7451		-
7452		- return $processed > 0 ? true : false;
7453		-
7454		- } catch (Exception $e) {
7455		- if (file_exists($pdf_path)) {
7456		- wp_delete_file($pdf_path);
7457		- }
7458		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
7459		- }
7460		-}
7461		-
7462		-/**
7463		- * Extract WooCommerce product content including pricing
7464		- *
7465		- * @param string $url The product URL
7466		- * @return string|false Product content with pricing, or false if not found
7467		- */
7468		-private function mxchat_extract_woocommerce_product_content($url) {
7469		- // Try to get product ID from URL
7470		- $product_id = url_to_postid($url);
7471		-
7472		- // If url_to_postid fails, try to extract from URL pattern
7473		- if (!$product_id) {
7474		- $product_slug = '';
7475		-
7476		- // Handle pretty permalinks: /product/product-name/
7477		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
7478		- $product_slug = $matches[1];
7479		- }
7480		-
7481		- if (!empty($product_slug)) {
7482		- $product_post = get_page_by_path($product_slug, OBJECT, 'product');
7483		- if ($product_post) {
7484		- $product_id = $product_post->ID;
7485		- }
7486		- }
7487		- }
7488		-
7489		- if (!$product_id) {
7490		- return false;
7491		- }
7492		-
7493		- // Get WooCommerce product object
7494		- $product = wc_get_product($product_id);
7495		-
7496		- if (!$product) {
7497		- return false;
7498		- }
7499		-
7500		- // Build product content with pricing (similar to mxchat_store_product_embedding)
7501		- $title = $product->get_name();
7502		- $description = $product->get_description();
7503		- $short_description = $product->get_short_description();
7504		- $sku = $product->get_sku();
7505		-
7506		- // Get pricing information
7507		- $regular_price = $product->get_regular_price();
7508		- $sale_price = $product->get_sale_price();
7509		- $price = $product->get_price(); // Current active price
7510		-
7511		- // Get currency symbol
7512		- $currency_symbol = get_woocommerce_currency_symbol();
7513		-
7514		- // Format content
7515		- $content = $title . "\n\n";
7516		-
7517		- if (!empty($short_description)) {
7518		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
7519		- }
7520		-
7521		- if (!empty($description)) {
7522		- $content .= wp_strip_all_tags($description) . "\n\n";
7523		- }
7524		-
7525		- // Add pricing information
7526		- if (!empty($regular_price)) {
7527		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
7528		- } elseif (!empty($price)) {
7529		- $content .= "Price: " . $currency_symbol . $price . "\n";
7530		- }
7531		-
7532		- if (!empty($sale_price) && $sale_price !== $regular_price) {
7533		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
7534		- }
7535		-
7536		- // Handle variable products - show price range
7537		- if ($product->is_type('variable')) {
7538		- $min_price = $product->get_variation_price('min');
7539		- $max_price = $product->get_variation_price('max');
7540		- if ($min_price !== $max_price) {
7541		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
7542		- }
7543		- }
7544		-
7545		- if (!empty($sku)) {
7546		- $content .= "SKU: " . $sku . "\n";
7547		- }
7548		-
7549		- // Get product categories
7550		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
7551		- if (!empty($categories) && !is_wp_error($categories)) {
7552		- $content .= "Categories: " . implode(', ', $categories) . "\n";
7553		- }
7554		-
7555		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
7556		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
7557		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
7558		- foreach ($custom_tabs as $tab) {
7559		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
7560		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
7561		-
7562		- if (!empty($tab_title) && !empty($tab_content)) {
7563		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
7564		- }
7565		- }
7566		- }
7567		-
7568		- // Also check for reusable/saved tabs applied to this product
7569		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
7570		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
7571		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
7572		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
7573		- foreach ($applied_saved_tabs as $saved_tab_id) {
7574		- if (isset($saved_tabs[$saved_tab_id])) {
7575		- $tab = $saved_tabs[$saved_tab_id];
7576		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
7577		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
7578		-
7579		- if (!empty($tab_title) && !empty($tab_content)) {
7580		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
7581		- }
7582		- }
7583		- }
7584		- }
7585		- }
7586		-
7587		- return $this->mxchat_sanitize_content_for_api($content);
7588		-}
7589		-
7590		-/**
7591		- * Process a PDF page from the queue
7592		- */
7593		-private function mxchat_process_queue_pdf_page($item_data, $bot_id = 'default') {
7594		- $pdf_path = isset($item_data['pdf_path']) ? $item_data['pdf_path'] : '';
7595		- $pdf_url = isset($item_data['pdf_url']) ? $item_data['pdf_url'] : '';
7596		- $page_number = isset($item_data['page_number']) ? absint($item_data['page_number']) : 0;
7597		- $total_pages = isset($item_data['total_pages']) ? absint($item_data['total_pages']) : 0;
7598		-
7599		- if (empty($pdf_path) || !file_exists($pdf_path)) {
7600		- return new WP_Error('pdf_not_found', 'PDF file not found: ' . $pdf_path);
7601		- }
7602		-
7603		- if ($page_number < 1) {
7604		- return new WP_Error('invalid_page', 'Invalid page number');
7605		- }
7606		-
7607		- try {
7608		- mxchat_load_pdf_parser();
7609		- $parser = new \Smalot\PdfParser\Parser();
7610		- $pdf = $parser->parseFile($pdf_path);
7611		- $pages = $pdf->getPages();
7612		-
7613		- if (!isset($pages[$page_number - 1])) {
7614		- return new WP_Error('page_not_found', 'Page ' . $page_number . ' not found in PDF');
7615		- }
7616		-
7617		- $text = $pages[$page_number - 1]->getText();
7618		-
7619		- if (empty($text)) {
7620		- return new WP_Error('empty_page', 'Page ' . $page_number . ': No text could be extracted — page may contain only images, links, or non-standard encoding');
7621		- }
7622		-
7623		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
7624		-
7625		- if (empty($sanitized)) {
7626		- return new WP_Error('empty_after_sanitization', 'Page ' . $page_number . ': Text was extracted but contained only special characters, control codes, or unsupported content that was removed during cleanup');
7627		- }
7628		-
7629		- // Create metadata
7630		- $metadata = array(
7631		- 'document_type' => 'pdf',
7632		- 'total_pages' => $total_pages,
7633		- 'current_page' => $page_number,
7634		- 'source_url' => $pdf_url
7635		- );
7636		-
7637		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
7638		- $page_url = esc_url($pdf_url . "#page=" . $page_number);
7639		-
7640		- // Get bot-specific API key
7641		- $bot_options = $this->get_bot_options($bot_id);
7642		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
7643		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
7644		-
7645		- if (strpos($selected_model, 'voyage') === 0) {
7646		- $api_key = $options['voyage_api_key'] ?? '';
7647		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
7648		- $api_key = $options['gemini_api_key'] ?? '';
7649		- } else {
7650		- $api_key = $options['api_key'] ?? '';
7651		- }
7652		-
7653		- if (empty($api_key)) {
7654		- return new WP_Error('no_api_key', 'API key not configured for bot: ' . $bot_id);
7655		- }
7656		-
7657		- // Submit to database - UPDATED 2.5.6: Added content_type 'pdf'
7658		- $result = MxChat_Utils::submit_content_to_db(
7659		- $content_with_metadata,
7660		- $page_url,
7661		- $api_key,
7662		- null,
7663		- $bot_id,
7664		- 'pdf'
7665		- );
7666		-
7667		- return $result;
7668		-
7669		- } catch (Exception $e) {
7670		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
7671		- }
7672		-}
7673		-
7674		-/**
7675		- * AJAX: Get queue processing status
7676		- */
7677		-public function ajax_mxchat_get_queue_status() {
7678		- // Verify nonce and permissions
7679		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7680		-
7681		- if (!current_user_can('manage_options')) {
7682		- wp_send_json_error('Unauthorized access');
7683		- }
7684		-
7685		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7686		-
7687		- if (empty($queue_id)) {
7688		- wp_send_json_error('Missing queue ID');
7689		- }
7690		-
7691		- global $wpdb;
7692		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7693		-
7694		- // Get counts by status
7695		- $counts = $wpdb->get_results($wpdb->prepare(
7696		- "SELECT status, COUNT(*) as count
7697		- FROM $table_name
7698		- WHERE queue_id = %s
7699		- GROUP BY status",
7700		- $queue_id
7701		- ), OBJECT_K);
7702		-
7703		- $total = 0;
7704		- $completed = 0;
7705		- $failed = 0;
7706		- $processing = 0;
7707		- $pending = 0;
7708		-
7709		- foreach ($counts as $status => $data) {
7710		- $count = absint($data->count);
7711		- $total += $count;
7712		-
7713		- switch ($status) {
7714		- case 'completed':
7715		- $completed = $count;
7716		- break;
7717		- case 'failed':
7718		- $failed = $count;
7719		- break;
7720		- case 'processing':
7721		- $processing = $count;
7722		- break;
7723		- case 'pending':
7724		- $pending = $count;
7725		- break;
7726		- }
7727		- }
7728		-
7729		- // Calculate percentage
7730		- $percentage = $total > 0 ? round((($completed + $failed) / $total) * 100) : 0;
7731		-
7732		- // Get failed items details (include all failed items, not just those that exhausted retries)
7733		- $failed_items = array();
7734		- if ($failed > 0) {
7735		- $failed_items = $wpdb->get_results($wpdb->prepare(
7736		- "SELECT item_type, item_data, error_message, attempts
7737		- FROM $table_name
7738		- WHERE queue_id = %s
7739		- AND status = 'failed'
7740		- ORDER BY id DESC
7741		- LIMIT 50",
7742		- $queue_id
7743		- ));
7744		- }
7745		-
7746		- // Get queue metadata
7747		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
7748		- $queue_type = $this->mxchat_get_queue_meta($queue_id, 'queue_type');
7749		-
7750		- // Determine if queue is complete
7751		- $is_complete = ($pending === 0 && $processing === 0);
7752		-
7753		- wp_send_json_success(array(
7754		- 'queue_id' => $queue_id,
7755		- 'queue_type' => $queue_type,
7756		- 'source_url' => $source_url,
7757		- 'total' => $total,
7758		- 'completed' => $completed,
7759		- 'failed' => $failed,
7760		- 'processing' => $processing,
7761		- 'pending' => $pending,
7762		- 'percentage' => $percentage,
7763		- 'is_complete' => $is_complete,
7764		- 'failed_items' => $failed_items,
7765		- 'status' => $is_complete ? 'complete' : 'processing'
7766		- ));
7767		-}
7768		-
7769		-/**
7770		- * AJAX: Clear completed queue
7771		- */
7772		-public function ajax_mxchat_clear_queue() {
7773		- // Verify nonce and permissions
7774		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7775		-
7776		- if (!current_user_can('manage_options')) {
7777		- wp_send_json_error('Unauthorized access');
7778		- }
7779		-
7780		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7781		-
7782		- if (empty($queue_id)) {
7783		- wp_send_json_error('Missing queue ID');
7784		- }
7785		-
7786		- global $wpdb;
7787		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7788		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7789		-
7790		- // Delete queue items
7791		- $wpdb->delete(
7792		- $table_name,
7793		- array('queue_id' => $queue_id),
7794		- array('%s')
7795		- );
7796		-
7797		- // Delete queue metadata
7798		- $wpdb->delete(
7799		- $meta_table,
7800		- array('queue_id' => $queue_id),
7801		- array('%s')
7802		- );
7803		-
7804		- wp_send_json_success(array(
7805		- 'message' => 'Queue cleared successfully'
7806		- ));
7807		-}
7808		-
7809		-/**
7810		- * AJAX: Retry failed items in queue
7811		- */
7812		-public function ajax_mxchat_retry_failed() {
7813		- // Verify nonce and permissions
7814		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7815		-
7816		- if (!current_user_can('manage_options')) {
7817		- wp_send_json_error('Unauthorized access');
7818		- }
7819		-
7820		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7821		-
7822		- if (empty($queue_id)) {
7823		- wp_send_json_error('Missing queue ID');
7824		- }
7825		-
7826		- global $wpdb;
7827		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7828		-
7829		- // Reset failed items to pending and reset attempt count
7830		- $updated = $wpdb->update(
7831		- $table_name,
7832		- array(
7833		- 'status' => 'pending',
7834		- 'attempts' => 0,
7835		- 'error_message' => null
7836		- ),
7837		- array(
7838		- 'queue_id' => $queue_id,
7839		- 'status' => 'failed'
7840		- ),
7841		- array('%s', '%d', '%s'),
7842		- array('%s', '%s')
7843		- );
7844		-
7845		- wp_send_json_success(array(
7846		- 'message' => 'Reset ' . $updated . ' failed items for retry',
7847		- 'reset_count' => $updated
7848		- ));
7849		-}
7850		-
7851		-
7852		-public function ajax_mxchat_mark_queue_complete() {
7853		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7854		-
7855		- if (!current_user_can('manage_options')) {
7856		- wp_send_json_error('Unauthorized access');
7857		- }
7858		-
7859		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7860		-
7861		- if (empty($queue_id)) {
7862		- wp_send_json_error('Missing queue ID');
7863		- }
7864		-
7865		- // Clear active queue transients
7866		- if (strpos($queue_id, 'sitemap_') === 0) {
7867		- delete_transient('mxchat_active_queue_sitemap');
7868		- } else if (strpos($queue_id, 'pdf_') === 0) {
7869		- delete_transient('mxchat_active_queue_pdf');
7870		- }
7871		-
7872		- wp_send_json_success(array('message' => 'Queue marked as complete'));
7873		-}
7874		-
7875	3346
7876	3347	// ========================================
7877	3348	// STATIC ACCESS METHODS
7878	3349	// ========================================