← All changes | admin/class-knowledge-manager.php +1612 -6410 3.2.6 → 2.2.3 View file →

		@@ -15,72 +15,49 @@
15	15
16	16	/**
17	17	* Constructor - Register hooks for content processing
18	18	*/
19		-public function __construct() {
20		- $this->options = get_option('mxchat_options', array());
21		- $this->mxchat_init_hooks();
	19	+ public function __construct() {
	20	+ $this->options = get_option('mxchat_options', array());
	21	+ $this->mxchat_init_hooks();
	22	+ }
22	23
23		- $this->mxchat_init_role_hooks();
24		-}
25		-
26		-/**
27		- * Initialize WordPress hooks for content processing
28		- *
29		- */
30		-private function mxchat_init_hooks() {
31		- // Admin post handlers for form submissions
32		- add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
33		- add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
34		- add_action('admin_post_mxchat_submit_pdf_file', array($this, 'mxchat_handle_pdf_file_submission'));
35		- add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
36		-
37		- // AJAX handlers for real-time processing and status updates
38		- add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
39		- add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status'));
40		- add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
41		- add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
42		- add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
43		- add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
44		- add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
45		- add_action('wp_ajax_mxchat_delete_chunks_by_url', array($this, 'ajax_mxchat_delete_chunks_by_url'));
46		- add_action('wp_ajax_mxchat_delete_wordpress_prompt', array($this, 'ajax_mxchat_delete_wordpress_prompt'));
47		- add_action('wp_ajax_mxchat_bulk_delete_knowledge', array($this, 'ajax_mxchat_bulk_delete_knowledge'));
48		- add_action('wp_ajax_mxchat_update_role_restriction', array($this, 'ajax_mxchat_update_role_restriction'));
49		-
50		- // Queue-based processing AJAX handlers
51		- add_action('wp_ajax_mxchat_get_next_queue_item', array($this, 'ajax_mxchat_get_next_queue_item'));
52		- add_action('wp_ajax_mxchat_process_queue_item', array($this, 'ajax_mxchat_process_queue_item'));
53		- add_action('wp_ajax_mxchat_get_queue_status', array($this, 'ajax_mxchat_get_queue_status'));
54		- add_action('wp_ajax_mxchat_clear_queue', array($this, 'ajax_mxchat_clear_queue'));
55		- add_action('wp_ajax_mxchat_retry_failed', array($this, 'ajax_mxchat_retry_failed'));
56		- add_action('wp_ajax_mxchat_get_recent_entries', array($this, 'ajax_mxchat_get_recent_entries'));
57		- add_action('wp_ajax_mxchat_detect_sitemaps', array($this, 'ajax_mxchat_detect_sitemaps'));
58		- add_action('wp_ajax_mxchat_refresh_pinecone_entries', array($this, 'ajax_mxchat_refresh_pinecone_entries'));
59		- add_action('wp_ajax_mxchat_paginate_entries', array($this, 'ajax_mxchat_paginate_entries'));
60		- add_action('wp_ajax_mxchat_get_entry_content', array($this, 'ajax_mxchat_get_entry_content'));
61		- add_action('wp_ajax_mxchat_save_entry_content', array($this, 'ajax_mxchat_save_entry_content'));
	24	+ /**
	25	+ * Initialize WordPress hooks for content processing
	26	+ */
	27	+ private function mxchat_init_hooks() {
	28	+ // Admin post handlers for form submissions
	29	+ add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
	30	+ add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
	31	+ add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
	32	+
	33	+ // AJAX handlers for real-time processing and status updates
	34	+ add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
	35	+ add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status')); // NEW
	36	+ add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
	37	+ add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
	38	+ add_action('wp_ajax_mxchat_manual_batch_process', array($this, 'ajax_manual_batch_process'));
	39	+ add_action('mxchat_delete_content', array($this, 'mxchat_delete_from_pinecone_by_url'), 10, 1);
62	40
63		- // WordPress post management hooks
64		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
65		- add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
66		- add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
67		- add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
	41	+
	42	+ // Cron handlers for background processing
	43	+ add_action('mxchat_process_sitemap_urls', array($this, 'mxchat_process_sitemap_urls_cron'), 10, 5);
	44	+ add_action('mxchat_process_pdf_pages', array($this, 'mxchat_process_pdf_pages_cron'), 10, 5);
	45	+
	46	+ // WordPress post management hooks
	47	+ add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
	48	+ add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
	49	+ add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
	50	+ add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
68	51
69		- // ACF hook - fires AFTER ACF fields are saved, ensuring ACF data is available
70		- // Priority 20 to run after ACF's own save (which runs at priority 10)
71		- add_action('acf/save_post', array($this, 'mxchat_handle_acf_save'), 20);
72		-
73		- add_action('wp_ajax_mxchat_mark_queue_complete', array($this, 'ajax_mxchat_mark_queue_complete'));
74		-
75		- // WooCommerce product hooks (if WooCommerce is active)
76		- if (class_exists('WooCommerce')) {
77		- add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
78		- add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
79		- add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
80		- add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
	52	+ // WooCommerce product hooks (if WooCommerce is active)
	53	+ if (class_exists('WooCommerce')) {
	54	+ add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
	55	+ add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
	56	+ add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
	57	+ }
	58	+
81	59	}
82		-}
83	60
84	61	/**
85	62	* Get current options (refreshed)
86	63	*/
		@@ -90,9 +67,315 @@
90	67	}
91	68	return $this->options;
92	69	}
93	70
	71	+
	72	+ /**
	73	+ * Handle manual batch processing via AJAX
	74	+ */
	75	+public function ajax_manual_batch_process() {
	76	+ try {
	77	+ // Verify nonce and permissions
	78	+ check_ajax_referer('mxchat_status_nonce', 'nonce');
	79	+
	80	+ if (!current_user_can('manage_options')) {
	81	+ wp_send_json_error('Unauthorized access');
	82	+ }
	83	+
	84	+ $process_type = sanitize_text_field($_POST['process_type'] ?? '');
	85	+ $url = sanitize_text_field($_POST['url'] ?? '');
	86	+
	87	+ if (empty($process_type) || empty($url)) {
	88	+ wp_send_json_error('Missing required parameters');
	89	+ }
	90	+
	91	+ $processed = 0;
	92	+
	93	+ if ($process_type === 'pdf') {
	94	+ $processed = $this->mxchat_manual_process_pdf_batch($url);
	95	+ } elseif ($process_type === 'sitemap') {
	96	+ $processed = $this->mxchat_manual_process_sitemap_batch($url);
	97	+ }
	98	+
	99	+ if ($processed > 0) {
	100	+ wp_send_json_success(array(
	101	+ 'message' => "Processed {$processed} items successfully",
	102	+ 'processed' => $processed
	103	+ ));
	104	+ } else {
	105	+ wp_send_json_error('No items were processed');
	106	+ }
	107	+
	108	+ } catch (Exception $e) {
	109	+ //error_log('Manual batch process error: ' . $e->getMessage());
	110	+ wp_send_json_error('Processing failed: ' . $e->getMessage());
	111	+ }
	112	+}
94	113
	114	+/**
	115	+ * Process a small PDF batch manually - DIRECT PROCESSING
	116	+ */
	117	+private function mxchat_manual_process_pdf_batch($pdf_url) {
	118	+ try {
	119	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	120	+ $status = get_transient($status_key);
	121	+
	122	+ if (!$status || $status['status'] !== 'processing') {
	123	+ //error_log('Manual PDF: No processing status found');
	124	+ return 0;
	125	+ }
	126	+
	127	+ //error_log('Manual PDF: Starting direct processing for ' . $pdf_url);
	128	+
	129	+ // Get current progress
	130	+ $current_page = $status['processed_pages'] ?? 0;
	131	+ $total_pages = $status['total_pages'] ?? 0;
	132	+
	133	+ if ($current_page >= $total_pages) {
	134	+ //error_log('Manual PDF: Already completed');
	135	+ return 0;
	136	+ }
	137	+
	138	+ // Try to download the PDF again for processing
	139	+ $response = wp_remote_get($pdf_url, array('timeout' => 30));
	140	+
	141	+ if (is_wp_error($response)) {
	142	+ //error_log('Manual PDF: Failed to download PDF: ' . $response->get_error_message());
	143	+ return 0;
	144	+ }
	145	+
	146	+ $pdf_content = wp_remote_retrieve_body($response);
	147	+ if (empty($pdf_content)) {
	148	+ //error_log('Manual PDF: Empty PDF content');
	149	+ return 0;
	150	+ }
	151	+
	152	+ // Save PDF temporarily
	153	+ $upload_dir = wp_upload_dir();
	154	+ $temp_pdf_path = trailingslashit($upload_dir['path']) . 'temp_manual_' . time() . '.pdf';
	155	+ file_put_contents($temp_pdf_path, $pdf_content);
	156	+
	157	+ // Process 2 pages directly
	158	+ $processed = $this->mxchat_process_pdf_pages_direct($temp_pdf_path, $pdf_url, $current_page, 5);
	159	+
	160	+ // Clean up temp file
	161	+ if (file_exists($temp_pdf_path)) {
	162	+ wp_delete_file($temp_pdf_path);
	163	+ }
	164	+
	165	+ //error_log('Manual PDF: Processed ' . $processed . ' pages');
	166	+ return $processed;
	167	+
	168	+ } catch (Exception $e) {
	169	+ //error_log('Manual PDF batch error: ' . $e->getMessage());
	170	+ return 0;
	171	+ }
	172	+}
	173	+
	174	+/**
	175	+ * Process PDF pages directly without cron
	176	+ */
	177	+private function mxchat_process_pdf_pages_direct($pdf_path, $pdf_url, $start_page, $batch_size) {
	178	+ try {
	179	+ if (!file_exists($pdf_path)) {
	180	+ //error_log('Direct PDF: File not found at ' . $pdf_path);
	181	+ return 0;
	182	+ }
	183	+
	184	+ $parser = new \Smalot\PdfParser\Parser();
	185	+ $pdf = $parser->parseFile($pdf_path);
	186	+ $pages = $pdf->getPages();
	187	+
	188	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	189	+ $status = get_transient($status_key);
	190	+
	191	+ if (!$status) {
	192	+ return 0;
	193	+ }
	194	+
	195	+ $options = get_option('mxchat_options');
	196	+ $api_key = $options['api_key'] ?? '';
	197	+
	198	+ if (empty($api_key)) {
	199	+ //error_log('Direct PDF: No API key');
	200	+ return 0;
	201	+ }
	202	+
	203	+ $processed = 0;
	204	+ $end_page = min($start_page + $batch_size, count($pages));
	205	+
	206	+ for ($i = $start_page; $i < $end_page; $i++) {
	207	+ try {
	208	+ $page_number = $i + 1;
	209	+ $text = $pages[$i]->getText();
	210	+
	211	+ if (empty($text)) {
	212	+ //error_log('Direct PDF: Empty text on page ' . $page_number);
	213	+ continue;
	214	+ }
	215	+
	216	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
	217	+ if (empty($sanitized_content)) {
	218	+ //error_log('Direct PDF: No content after sanitization on page ' . $page_number);
	219	+ continue;
	220	+ }
	221	+
	222	+ // Generate embedding
	223	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	224	+ if (is_string($embedding_vector)) {
	225	+ //error_log('Direct PDF: Embedding failed on page ' . $page_number . ': ' . $embedding_vector);
	226	+ continue;
	227	+ }
	228	+
	229	+ // Create metadata
	230	+ $metadata = array(
	231	+ 'document_type' => 'pdf',
	232	+ 'total_pages' => count($pages),
	233	+ 'current_page' => $page_number,
	234	+ 'source_url' => $pdf_url
	235	+ );
	236	+
	237	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	238	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
	239	+
	240	+ // Store in database
	241	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $api_key);
	242	+
	243	+ if (is_wp_error($db_result)) {
	244	+ //error_log('Direct PDF: DB error on page ' . $page_number . ': ' . $db_result->get_error_message());
	245	+ continue;
	246	+ }
	247	+
	248	+ $processed++;
	249	+ //error_log('Direct PDF: Successfully processed page ' . $page_number);
	250	+
	251	+ // Update status
	252	+ $status['processed_pages'] = $i + 1;
	253	+ $status['last_update'] = time();
	254	+ $status['percentage'] = round(($status['processed_pages'] / $status['total_pages']) * 100);
	255	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	256	+
	257	+ } catch (Exception $e) {
	258	+ //error_log('Direct PDF: Error processing page ' . ($i + 1) . ': ' . $e->getMessage());
	259	+ continue;
	260	+ }
	261	+ }
	262	+
	263	+ // Check if completed
	264	+ if ($status['processed_pages'] >= $status['total_pages']) {
	265	+ $status['status'] = 'complete';
	266	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	267	+ //error_log('Direct PDF: Processing completed');
	268	+ }
	269	+
	270	+ return $processed;
	271	+
	272	+ } catch (Exception $e) {
	273	+ //error_log('Direct PDF processing error: ' . $e->getMessage());
	274	+ return 0;
	275	+ }
	276	+}
	277	+
	278	+/**
	279	+ * Process a small sitemap batch manually - DIRECT PROCESSING
	280	+ */
	281	+private function mxchat_manual_process_sitemap_batch($sitemap_url) {
	282	+ try {
	283	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	284	+ $status = get_transient($status_key);
	285	+
	286	+ if (!$status || $status['status'] !== 'processing') {
	287	+ return 0;
	288	+ }
	289	+
	290	+ //error_log('Manual Sitemap: Starting direct processing for ' . $sitemap_url);
	291	+
	292	+ // Re-fetch the sitemap to get URLs
	293	+ $response = wp_remote_get($sitemap_url, array('timeout' => 30));
	294	+ if (is_wp_error($response)) {
	295	+ //error_log('Manual Sitemap: Failed to fetch sitemap');
	296	+ return 0;
	297	+ }
	298	+
	299	+ $sitemap_content = wp_remote_retrieve_body($response);
	300	+ $xml = simplexml_load_string($sitemap_content);
	301	+
	302	+ if (!$xml) {
	303	+ //error_log('Manual Sitemap: Invalid XML');
	304	+ return 0;
	305	+ }
	306	+
	307	+ $urls = array();
	308	+ foreach ($xml->url as $url_element) {
	309	+ $urls[] = (string)$url_element->loc;
	310	+ }
	311	+
	312	+ $current_processed = $status['processed_urls'] ?? 0;
	313	+ $batch_size = 5;
	314	+ $processed = 0;
	315	+
	316	+ // Process next 2 URLs
	317	+ for ($i = $current_processed; $i < min($current_processed + $batch_size, count($urls)); $i++) {
	318	+ $url = $urls[$i];
	319	+
	320	+ if ($this->mxchat_process_single_url_direct($url)) {
	321	+ $processed++;
	322	+ }
	323	+
	324	+ // Update status
	325	+ $status['processed_urls'] = $i + 1;
	326	+ $status['last_update'] = time();
	327	+ $status['percentage'] = round(($status['processed_urls'] / $status['total_urls']) * 100);
	328	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	329	+ }
	330	+
	331	+ // Check if completed
	332	+ if ($status['processed_urls'] >= $status['total_urls']) {
	333	+ $status['status'] = 'complete';
	334	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	335	+ }
	336	+
	337	+ //error_log('Manual Sitemap: Processed ' . $processed . ' URLs');
	338	+ return $processed;
	339	+
	340	+ } catch (Exception $e) {
	341	+ //error_log('Manual sitemap batch error: ' . $e->getMessage());
	342	+ return 0;
	343	+ }
	344	+}
	345	+
	346	+/**
	347	+ * Process a single URL directly
	348	+ */
	349	+private function mxchat_process_single_url_direct($url) {
	350	+ try {
	351	+ $response = wp_remote_get($url, array('timeout' => 30));
	352	+ if (is_wp_error($response)) {
	353	+ return false;
	354	+ }
	355	+
	356	+ $html = wp_remote_retrieve_body($response);
	357	+ $content = $this->mxchat_extract_main_content($html);
	358	+ $sanitized = $this->mxchat_sanitize_content_for_api($content);
	359	+
	360	+ if (empty($sanitized)) {
	361	+ return false;
	362	+ }
	363	+
	364	+ $options = get_option('mxchat_options');
	365	+ $api_key = $options['api_key'] ?? '';
	366	+
	367	+ $result = MxChat_Utils::submit_content_to_db($sanitized, $url, $api_key);
	368	+
	369	+ return !is_wp_error($result);
	370	+
	371	+ } catch (Exception $e) {
	372	+ //error_log('Single URL processing error: ' . $e->getMessage());
	373	+ return false;
	374	+ }
	375	+}
	376	+
	377	+
95	378	// ========================================
96	379	// MAIN CONTENT SUBMISSION HANDLERS
97	380	// ========================================
98	381
		@@ -108,18 +391,13 @@
108	391	wp_die(esc_html__('Nonce verification failed.', 'mxchat'));
109	392	}
110	393
111	394	// Sanitize the inputs.
112		- // Use wp_kses_post to allow safe HTML and wp_unslash to remove WordPress added slashes
113		- $article_content = wp_kses_post(wp_unslash($_POST['article_content']));
	395	+ $article_content = sanitize_textarea_field($_POST['article_content']);
114	396	$article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
115	397
116		- // Get bot_id from form submission
117		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
118		-
119		- // Get bot-specific options and API key
120		- $bot_options = $this->get_bot_options($bot_id);
121		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	398	+ // Get API key for submission
	399	+ $options = get_option('mxchat_options');
122	400	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
123	401
124	402	if (strpos($selected_model, 'voyage') === 0) {
125	403	$api_key = $options['voyage_api_key'] ?? '';
		@@ -137,10 +415,10 @@
137	415	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
138	416	exit;
139	417	}
140	418
141		- // Use centralized utility function with bot_id
142		- $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key, null, $bot_id);
	419	+ // Use centralized utility function for storage
	420	+ $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key);
143	421
144	422	if (is_wp_error($result)) {
145	423	set_transient('mxchat_admin_notice_error',
146	424	esc_html__('Error storing content: ', 'mxchat') . $result->get_error_message(),
		@@ -155,23 +433,17 @@
155	433
156	434	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
157	435	exit;
158	436	}
159		-
160	437	public function mxchat_is_pdf_url($url, $response) {
161	438	$content_type = wp_remote_retrieve_header($response, 'content-type');
162	439	$file_extension = strtolower(pathinfo($url, PATHINFO_EXTENSION));
163	440
164		- // Check Content-Disposition header for .pdf filename (Google Drive sends this)
165		- $disposition = wp_remote_retrieve_header($response, 'content-disposition');
166		- $has_pdf_disposition = ! empty($disposition) && stripos($disposition, '.pdf') !== false;
167		-
168		- return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf' || $has_pdf_disposition;
	441	+ return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf';
169	442	}
170		-
171		-
172		-public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response, $bot_id = 'default') {
	443	+public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response) {
173	444	if (!current_user_can('manage_options')) {
	445	+ //error_log(esc_html__('Unauthorized PDF processing attempt', 'mxchat'));
174	446	return false;
175	447	}
176	448
177	449	$pdf_url = esc_url_raw($pdf_url);
		@@ -177,8 +449,9 @@
177	449	$pdf_url = esc_url_raw($pdf_url);
178	450	$upload_dir = wp_upload_dir();
179	451
180	452	if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
	453	+ //error_log(sprintf(esc_html__('Upload directory error: %s', 'mxchat'), esc_html($upload_dir['error'])));
181	454	return false;
182	455	}
183	456
184	457	$pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
		@@ -185,12 +458,14 @@
185	458	$pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
186	459
187	460	$response_body = wp_remote_retrieve_body($response);
188	461	if (empty($response_body)) {
	462	+ //error_log(esc_html__('Empty PDF response body', 'mxchat'));
189	463	return false;
190	464	}
191	465
192	466	if (!wp_mkdir_p(dirname($pdf_path))) {
	467	+ //error_log(sprintf(esc_html__('Failed to create directory for PDF: %s', 'mxchat'), esc_html($pdf_path)));
193	468	return false;
194	469	}
195	470
196	471	try {
		@@ -199,693 +474,295 @@
199	474	if (!file_exists($pdf_path)) {
200	475	throw new Exception(__('Failed to save PDF file', 'mxchat'));
201	476	}
202	477
203		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
204		-
205		- if ($total_pages === false || $total_pages < 1) {
206		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
207		- }
	478	+ $parser = new \Smalot\PdfParser\Parser();
	479	+ $pdf = $parser->parseFile($pdf_path);
	480	+ $total_pages = absint(count($pdf->getPages()));
208	481
209		- // Create unique queue ID
210		- $queue_id = 'pdf_' . md5($pdf_url . time());
211		-
212		- // Create array of pages to process
213		- $pages = array();
214		- for ($i = 1; $i <= $total_pages; $i++) {
215		- $pages[] = array(
216		- 'pdf_path' => $pdf_path,
217		- 'pdf_url' => $pdf_url,
218		- 'page_number' => $i,
219		- 'total_pages' => $total_pages
220		- );
	482	+ if ($total_pages < 1) {
	483	+ throw new Exception(__('Invalid PDF: no pages found', 'mxchat'));
221	484	}
222	485
223		- // Add pages to queue
224		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
	486	+ wp_schedule_single_event(time(), 'mxchat_process_pdf_pages', array(
	487	+ 'pdf_path' => $pdf_path,
	488	+ 'pdf_url' => $pdf_url,
	489	+ 'total_pages' => $total_pages,
	490	+ 'batch_size' => absint(15),
	491	+ 'batch_pause' => absint(10)
	492	+ ));
225	493
226		- if ($queued_count === 0) {
227		- wp_delete_file($pdf_path);
228		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
229		- }
	494	+ $status_data = array(
	495	+ 'total_pages' => $total_pages,
	496	+ 'processed_pages' => 0,
	497	+ 'status' => 'processing',
	498	+ 'last_update' => time()
	499	+ );
230	500
231		- // Store queue metadata
232		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $pdf_url);
233		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
234		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
235		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
236		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
237		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	501	+ set_transient(
	502	+ sanitize_key('mxchat_pdf_status_' . md5($pdf_url)),
	503	+ array_map('sanitize_text_field', $status_data),
	504	+ DAY_IN_SECONDS
	505	+ );
238	506
239		- // Store queue ID in transient for status tracking
240		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
241		- set_transient('mxchat_last_pdf_url', $pdf_url, DAY_IN_SECONDS);
	507	+ return __('scheduled', 'mxchat');
242	508
243		- return 'queued';
244		-
245	509	} catch (Exception $e) {
	510	+ //error_log(sprintf(esc_html__('Error preparing PDF for processing: %s', 'mxchat'), esc_html($e->getMessage())));
246	511	if (file_exists($pdf_path)) {
247	512	wp_delete_file($pdf_path);
248	513	}
249		- return $e->getMessage();
	514	+ return false;
250	515	}
251	516	}
252	517
253		-/**
254		- * Handle direct PDF file upload from the knowledge base page
255		- */
256		-public function mxchat_handle_pdf_file_submission() {
257		- if (!isset($_POST['submit_pdf_file']) || !current_user_can('manage_options')) {
258		- wp_die(esc_html__('Unauthorized access', 'mxchat'));
259		- }
	518	+public function mxchat_process_pdf_pages_cron($pdf_path, $pdf_url, $total_pages, $batch_size, $batch_pause) {
	519	+ // Validate inputs
	520	+ $pdf_path = sanitize_text_field($pdf_path);
	521	+ $pdf_url = esc_url_raw($pdf_url);
	522	+ $total_pages = absint($total_pages);
	523	+ $batch_size = absint($batch_size);
	524	+ $batch_pause = absint($batch_pause);
260	525
261		- check_admin_referer('mxchat_submit_pdf_file_action', 'mxchat_submit_pdf_file_nonce');
262		-
263		- $redirect_url = admin_url('admin.php?page=mxchat-prompts');
264		-
265		- // Validate file upload
266		- if (empty($_FILES['pdf_file']) || $_FILES['pdf_file']['error'] !== UPLOAD_ERR_OK) {
267		- $error_code = isset($_FILES['pdf_file']['error']) ? $_FILES['pdf_file']['error'] : UPLOAD_ERR_NO_FILE;
268		- $error_messages = array(
269		- UPLOAD_ERR_INI_SIZE => __('The uploaded file exceeds the server upload_max_filesize limit.', 'mxchat'),
270		- UPLOAD_ERR_FORM_SIZE => __('The uploaded file exceeds the form MAX_FILE_SIZE limit.', 'mxchat'),
271		- UPLOAD_ERR_PARTIAL => __('The file was only partially uploaded.', 'mxchat'),
272		- UPLOAD_ERR_NO_FILE => __('No file was uploaded. Please select a PDF file.', 'mxchat'),
273		- UPLOAD_ERR_NO_TMP_DIR => __('Server missing temporary folder.', 'mxchat'),
274		- UPLOAD_ERR_CANT_WRITE => __('Server failed to write file to disk.', 'mxchat'),
275		- );
276		- $error_msg = isset($error_messages[$error_code]) ? $error_messages[$error_code] : __('Unknown upload error.', 'mxchat');
277		- set_transient('mxchat_admin_notice_error', $error_msg, 30);
278		- wp_safe_redirect(esc_url($redirect_url));
279		- exit;
280		- }
281		-
282		- $file = $_FILES['pdf_file'];
283		-
284		- // Validate MIME type
285		- $finfo = finfo_open(FILEINFO_MIME_TYPE);
286		- $mime_type = finfo_file($finfo, $file['tmp_name']);
287		- finfo_close($finfo);
288		-
289		- if ($mime_type !== 'application/pdf') {
290		- set_transient('mxchat_admin_notice_error',
291		- esc_html__('Invalid file type. Only PDF files are accepted.', 'mxchat'),
292		- 30
293		- );
294		- wp_safe_redirect(esc_url($redirect_url));
295		- exit;
296		- }
297		-
298		- // Validate extension
299		- $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
300		- if ($ext !== 'pdf') {
301		- set_transient('mxchat_admin_notice_error',
302		- esc_html__('Invalid file extension. Only .pdf files are accepted.', 'mxchat'),
303		- 30
304		- );
305		- wp_safe_redirect(esc_url($redirect_url));
306		- exit;
307		- }
308		-
309		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
310		- $original_filename = sanitize_file_name($file['name']);
311		-
312		- $upload_dir = wp_upload_dir();
313		- if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
314		- set_transient('mxchat_admin_notice_error',
315		- esc_html__('WordPress upload directory is not writable.', 'mxchat'),
316		- 30
317		- );
318		- wp_safe_redirect(esc_url($redirect_url));
319		- exit;
320		- }
321		-
322		- $pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
323		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
324		-
325		- if (!wp_mkdir_p(dirname($pdf_path))) {
326		- set_transient('mxchat_admin_notice_error',
327		- esc_html__('Failed to create upload directory.', 'mxchat'),
328		- 30
329		- );
330		- wp_safe_redirect(esc_url($redirect_url));
331		- exit;
332		- }
333		-
334		- // Move uploaded file
335		- if (!move_uploaded_file($file['tmp_name'], $pdf_path)) {
336		- set_transient('mxchat_admin_notice_error',
337		- esc_html__('Failed to save uploaded PDF file.', 'mxchat'),
338		- 30
339		- );
340		- wp_safe_redirect(esc_url($redirect_url));
341		- exit;
342		- }
343		-
344	526	try {
345		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
346		-
347		- if ($total_pages === false || $total_pages < 1) {
348		- throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
	527	+ if (!file_exists($pdf_path)) {
	528	+ throw new Exception(sprintf('PDF file not found at path: %s', esc_html($pdf_path)));
349	529	}
350	530
351		- // Use original filename as the source identifier
352		- $source_label = 'upload://' . $original_filename;
353		-
354		- $queue_id = 'pdf_' . md5($source_label . time());
355		-
356		- $pages = array();
357		- for ($i = 1; $i <= $total_pages; $i++) {
358		- $pages[] = array(
359		- 'pdf_path' => $pdf_path,
360		- 'pdf_url' => $source_label,
361		- 'page_number' => $i,
362		- 'total_pages' => $total_pages,
363		- );
364		- }
365		-
366		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
367		-
368		- if ($queued_count === 0) {
369		- wp_delete_file($pdf_path);
370		- throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
371		- }
372		-
373		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $source_label);
374		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
375		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
376		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
377		- $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
378		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
379		-
380		- set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
381		- set_transient('mxchat_last_pdf_url', $source_label, DAY_IN_SECONDS);
382		-
383		- set_transient('mxchat_admin_notice_success',
384		- sprintf(
385		- esc_html__('PDF "%s" (%d pages) queued for processing. Processing will start automatically.', 'mxchat'),
386		- esc_html($original_filename),
387		- $total_pages
388		- ),
389		- 30
390		- );
391		-
392		- } catch (Exception $e) {
393		- if (file_exists($pdf_path)) {
394		- wp_delete_file($pdf_path);
395		- }
396		- set_transient('mxchat_admin_notice_error',
397		- esc_html__('Failed to process uploaded PDF: ', 'mxchat') . esc_html($e->getMessage()),
398		- 30
399		- );
400		- }
401		-
402		- wp_safe_redirect(esc_url($redirect_url));
403		- exit;
404		-}
405		-
406		-/**
407		- * Validate PDF and count pages with multiple parser attempts
408		- */
409		-private function mxchat_validate_and_count_pdf_pages($pdf_path) {
410		- // Method 1: Try with Smalot PDF Parser (your current method)
411		- try {
412		- mxchat_load_pdf_parser();
413	531	$parser = new \Smalot\PdfParser\Parser();
414	532	$pdf = $parser->parseFile($pdf_path);
415	533	$pages = $pdf->getPages();
416		- $page_count = count($pages);
417		-
418		- if ($page_count > 0) {
419		- //error_log('PDF parsed successfully with Smalot parser: ' . $page_count . ' pages');
420		- return $page_count;
421		- }
422		- } catch (Exception $e) {
423		- //error_log('Smalot PDF parser failed: ' . $e->getMessage());
424		- }
425	534
426		- // Method 2: Try with pdfinfo command (if available)
427		- if (function_exists('shell_exec') && !$this->mxchat_is_shell_disabled()) {
428		- try {
429		- $command = 'pdfinfo ' . escapeshellarg($pdf_path) . ' 2>&1';
430		- $output = shell_exec($command);
431		-
432		- if ($output && preg_match('/Pages:\s*(\d+)/', $output, $matches)) {
433		- $page_count = intval($matches[1]);
434		- if ($page_count > 0) {
435		- //error_log('PDF parsed successfully with pdfinfo: ' . $page_count . ' pages');
436		- return $page_count;
437		- }
438		- }
439		- } catch (Exception $e) {
440		- //error_log('pdfinfo command failed: ' . $e->getMessage());
	535	+ // Get current progress
	536	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	537	+ $status = get_transient($status_key);
	538	+
	539	+ if (!$status || !is_array($status)) {
	540	+ throw new Exception('Invalid status data retrieved from transient');
441	541	}
442		- }
443	542
444		- // Method 3: Try to repair PDF and parse again
445		- try {
446		- $repaired_path = $this->mxchat_attempt_pdf_repair($pdf_path);
447		- if ($repaired_path && $repaired_path !== $pdf_path) {
448		- mxchat_load_pdf_parser();
449		- $parser = new \Smalot\PdfParser\Parser();
450		- $pdf = $parser->parseFile($repaired_path);
451		- $pages = $pdf->getPages();
452		- $page_count = count($pages);
453		-
454		- if ($page_count > 0) {
455		- // Replace original with repaired version
456		- copy($repaired_path, $pdf_path);
457		- unlink($repaired_path);
458		- //error_log('PDF repaired and parsed successfully: ' . $page_count . ' pages');
459		- return $page_count;
460		- }
461		-
462		- // Clean up repaired file if it didn't work
463		- unlink($repaired_path);
	543	+ // Initialize failed pages list if it doesn't exist
	544	+ if (!isset($status['failed_pages_list']) || !is_array($status['failed_pages_list'])) {
	545	+ $status['failed_pages_list'] = [];
464	546	}
465		- } catch (Exception $e) {
466		- //error_log('PDF repair attempt failed: ' . $e->getMessage());
467		- }
468	547
469		- // Method 4: Manual PDF structure analysis (basic page count)
470		- try {
471		- $page_count = $this->mxchat_manual_pdf_page_count($pdf_path);
472		- if ($page_count > 0) {
473		- //error_log('PDF page count determined manually: ' . $page_count . ' pages');
474		- return $page_count;
	548	+ $start_page = absint($status['processed_pages']);
	549	+ $end_page = min($start_page + $batch_size, $total_pages);
	550	+ $options = get_option('mxchat_options');
	551	+
	552	+ if (empty($options['api_key'])) {
	553	+ throw new Exception('API key is missing or invalid');
475	554	}
476		- } catch (Exception $e) {
477		- //error_log('Manual PDF analysis failed: ' . $e->getMessage());
478		- }
479	555
480		- //error_log('All PDF parsing methods failed for: ' . $pdf_path);
481		- return false;
482		-}
	556	+ $successful_pages = 0;
	557	+ $failed_pages = 0;
483	558
484		-/**
485		- * Check if shell_exec is disabled
486		- */
487		-private function mxchat_is_shell_disabled() {
488		- $disabled = explode(',', ini_get('disable_functions'));
489		- return in_array('shell_exec', $disabled);
490		-}
	559	+ for ($i = $start_page; $i < $end_page; $i++) {
	560	+ $page_number = $i + 1;
	561	+ $max_retries = 3;
	562	+ $retry_count = 0;
	563	+ $page_processed = false;
	564	+ $last_error = '';
491	565
492		-/**
493		- * Attempt to repair PDF using basic methods
494		- */
495		-private function mxchat_attempt_pdf_repair($pdf_path) {
496		- try {
497		- $content = file_get_contents($pdf_path);
498		- if (!$content) {
499		- return false;
500		- }
	566	+ while (!$page_processed && $retry_count < $max_retries) {
	567	+ try {
	568	+ $text = $pages[$i]->getText();
	569	+
	570	+ if (empty($text)) {
	571	+ throw new Exception("Empty text on page {$page_number}");
	572	+ }
	573	+
	574	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
501	575
502		- // Check if PDF starts with proper header
503		- if (substr($content, 0, 4) !== '%PDF') {
504		- // Try to find PDF header in the content
505		- $header_pos = strpos($content, '%PDF');
506		- if ($header_pos !== false && $header_pos < 1024) {
507		- // Remove junk before PDF header
508		- $content = substr($content, $header_pos);
509		- $repaired_path = $pdf_path . '.repaired';
510		- file_put_contents($repaired_path, $content);
511		- return $repaired_path;
512		- }
513		- }
	576	+ if (empty($sanitized_content)) {
	577	+ throw new Exception("No valid content after sanitization on page {$page_number}");
	578	+ }
514	579
515		- // Check for EOF marker
516		- $content = rtrim($content);
517		- if (!preg_match('/%%EOF\s*$/', $content)) {
518		- // Add EOF marker if missing
519		- $content .= "\n%%EOF";
520		- $repaired_path = $pdf_path . '.repaired';
521		- file_put_contents($repaired_path, $content);
522		- return $repaired_path;
523		- }
	580	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	581	+
	582	+ if (is_string($embedding_vector)) {
	583	+ throw new Exception("Embedding generation failed: " . $embedding_vector);
	584	+ }
	585	+
	586	+ if (!is_array($embedding_vector)) {
	587	+ throw new Exception("Embedding generation returned unexpected result type: " . gettype($embedding_vector));
	588	+ }
	589	+
	590	+ $metadata = array(
	591	+ 'document_type' => 'pdf',
	592	+ 'total_pages' => $total_pages,
	593	+ 'current_page' => $page_number,
	594	+ 'prev_page' => $i > 0 ? $i : null,
	595	+ 'next_page' => $i < ($total_pages - 1) ? $i + 2 : null,
	596	+ 'source_url' => $pdf_url
	597	+ );
524	598
525		- } catch (Exception $e) {
526		- //error_log('PDF repair error: ' . $e->getMessage());
527		- }
	599	+ $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
	600	+ $page_url = esc_url($pdf_url . "#page=" . $page_number);
528	601
529		- return false;
530		-}
	602	+ $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $options['api_key']);
	603	+
	604	+ if (is_wp_error($db_result)) {
	605	+ throw new Exception("Database submission failed: " . $db_result->get_error_message());
	606	+ }
531	607
532		-/**
533		- * Manual PDF page counting by analyzing PDF structure
534		- */
535		-private function mxchat_manual_pdf_page_count($pdf_path) {
536		- try {
537		- $content = file_get_contents($pdf_path);
538		- if (!$content) {
539		- return 0;
540		- }
	608	+ // Success!
	609	+ $page_processed = true;
	610	+ $successful_pages++;
	611	+
	612	+ } catch (Exception $e) {
	613	+ $retry_count++;
	614	+ $last_error = $e->getMessage();
	615	+
	616	+ //error_log("PDF page {$page_number} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	617	+
	618	+ if ($retry_count < $max_retries) {
	619	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	620	+ sleep(pow(2, $retry_count - 1));
	621	+ }
	622	+ }
	623	+ }
541	624
542		- // Method 1: Count /Type /Page objects
543		- $page_count = preg_match_all('/\/Type\s*\/Page[^s]/', $content);
544		- if ($page_count > 0) {
545		- return $page_count;
546		- }
	625	+ // If page still not processed after all retries, mark as failed
	626	+ if (!$page_processed) {
	627	+ $failed_pages++;
	628	+ $status['failed_pages_list'][] = [
	629	+ 'page' => $page_number,
	630	+ 'error' => $last_error,
	631	+ 'time' => time(),
	632	+ 'retries' => $max_retries
	633	+ ];
	634	+
	635	+ // Limit failed pages list to prevent memory issues
	636	+ if (count($status['failed_pages_list']) > 50) {
	637	+ $status['failed_pages_list'] = array_slice($status['failed_pages_list'], -50);
	638	+ }
	639	+
	640	+ //error_log("PDF page {$page_number} permanently failed after {$max_retries} attempts: " . $last_error);
	641	+ }
547	642
548		- // Method 2: Look for /Count in pages object
549		- if (preg_match('/\/Type\s*\/Pages.*?\/Count\s+(\d+)/', $content, $matches)) {
550		- return intval($matches[1]);
	643	+ // Update progress
	644	+ $status['processed_pages'] = absint($page_number);
	645	+ $status['last_update'] = time();
	646	+ $status['failed_pages'] = absint($status['failed_pages'] ?? 0) + ($page_processed ? 0 : 1);
	647	+
	648	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
551	649	}
552	650
553		- // Method 3: Count page references
554		- $page_count = preg_match_all('/\d+\s+0\s+obj\s*<<[^>]*\/Type\s*\/Page/', $content);
555		- if ($page_count > 0) {
556		- return $page_count;
557		- }
558		-
559		- } catch (Exception $e) {
560		- //error_log('Manual PDF analysis error: ' . $e->getMessage());
561		- }
562		-
563		- return 0;
564		-}
565		-
566		-
567		-public function mxchat_save_inline_prompt() {
568		- // DEBUG: Log what we're receiving
569		- //error_log('=== MXCHAT DEBUG ===');
570		- //error_log('POST data: ' . print_r($_POST, true));
571		- //error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
572		-
573		- // Check for nonce security
574		- check_ajax_referer('mxchat_save_inline_nonce', '_ajax_nonce');
575		-
576		- // If we get here, nonce passed
577		- //error_log('Nonce verification PASSED');
578		-
579		- // Verify permissions
580		- if (!current_user_can('manage_options')) {
581		- wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
582		- return;
583		- }
584		-
585		- global $wpdb;
586		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
587		-
588		- // Validate and sanitize input data
589		- $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
590		- $article_content = isset($_POST['article_content']) ? wp_kses_post(wp_unslash($_POST['article_content'])) : '';
591		- $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
592		-
593		- if ($prompt_id > 0 && !empty($article_content)) {
594		- // Re-generate the embedding vector for the updated content
595		- $embedding_vector = $this->mxchat_generate_embedding($article_content);
596		- if (is_array($embedding_vector)) {
597		- // Serialize the embedding vector before storing it
598		- $embedding_vector_serialized = serialize($embedding_vector);
599		- // Update the prompt in the database
600		- $updated = $wpdb->update(
601		- $table_name,
602		- array(
603		- 'article_content' => $article_content,
604		- 'embedding_vector' => $embedding_vector_serialized,
605		- 'source_url' => $article_url,
606		- ),
607		- array('id' => $prompt_id),
608		- array('%s', '%s', '%s'),
609		- array('%d')
610		- );
611		- if ($updated !== false) {
612		- wp_send_json_success();
613		- } else {
614		- MxChat_Admin::mxchat_log_debug('knowledge_error', 'Database update failed when saving knowledge entry');
615		- wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
616		- }
	651	+ // Schedule next batch if needed
	652	+ if ($end_page < $total_pages) {
	653	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_pdf_pages', array(
	654	+ 'pdf_path' => $pdf_path,
	655	+ 'pdf_url' => $pdf_url,
	656	+ 'total_pages' => $total_pages,
	657	+ 'batch_size' => $batch_size,
	658	+ 'batch_pause' => $batch_pause
	659	+ ));
617	660	} else {
618		- MxChat_Admin::mxchat_log_debug('embedding_error', 'Embedding generation failed for knowledge entry');
619		- wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
620		- }
621		- } else {
622		- wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
623		- }
624		-}
625		-
626		-
627		-/**
628		- * AJAX: Get full content for editing — reassembles chunks if needed.
629		- * Works for both WordPress DB and Pinecone entries.
630		- */
631		-public function ajax_mxchat_get_entry_content() {
632		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
633		-
634		- if ( ! current_user_can('manage_options') ) {
635		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
636		- }
637		-
638		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
639		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
640		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
641		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
642		-
643		- if ( $data_source === 'pinecone' ) {
644		- // Pinecone: fetch vectors by source_url, reassemble chunks
645		- $content = $this->get_pinecone_entry_content( $source_url, $entry_id, $bot_id );
646		- } else {
647		- // WordPress DB
648		- $content = $this->get_wordpress_entry_content( $source_url, $entry_id );
649		- }
650		-
651		- if ( is_wp_error( $content ) ) {
652		- wp_send_json_error( array( 'message' => $content->get_error_message() ) );
653		- }
654		-
655		- wp_send_json_success( $content );
656		-}
657		-
658		-/**
659		- * Get content from WordPress DB — reassembles chunks by source_url.
660		- */
661		-private function get_wordpress_entry_content( $source_url, $entry_id ) {
662		- global $wpdb;
663		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
664		-
665		- // If we have a source_url, check for chunks
666		- if ( ! empty( $source_url ) && strpos( $source_url, 'mxchat://' ) !== 0 ) {
667		- $rows = $wpdb->get_results( $wpdb->prepare(
668		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
669		- $source_url
670		- ) );
671		-
672		- if ( $rows && count( $rows ) > 1 ) {
673		- // Multiple rows = chunked. Reassemble.
674		- $chunks = array();
675		- foreach ( $rows as $row ) {
676		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
677		- $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
678		- $chunks[ $index ] = $parsed['text'];
	661	+ // Processing complete
	662	+ $status['status'] = 'complete';
	663	+ $status['processed_pages'] = $total_pages;
	664	+
	665	+ // Add completion summary
	666	+ $status['completion_summary'] = [
	667	+ 'total_pages' => $total_pages,
	668	+ 'successful_pages' => $total_pages - absint($status['failed_pages'] ?? 0),
	669	+ 'failed_pages' => absint($status['failed_pages'] ?? 0),
	670	+ 'completion_time' => current_time('mysql')
	671	+ ];
	672	+
	673	+ // Save the completed status (don't delete it - let user dismiss manually)
	674	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	675	+
	676	+ // Clean up the temporary PDF file
	677	+ if (file_exists($pdf_path)) {
	678	+ wp_delete_file($pdf_path);
679	679	}
680		- ksort( $chunks );
681		- return array(
682		- 'content' => implode( "\n\n", $chunks ),
683		- 'source_url' => $source_url,
684		- 'is_chunked' => true,
685		- 'chunk_count' => count( $chunks ),
686		- 'content_type' => $rows[0]->content_type,
687		- );
688		- } elseif ( $rows && count( $rows ) === 1 ) {
689		- $parsed = MxChat_Chunker::parse_stored_chunk( $rows[0]->article_content );
690		- return array(
691		- 'content' => $parsed['text'],
692		- 'source_url' => $source_url,
693		- 'entry_id' => $rows[0]->id,
694		- 'is_chunked' => false,
695		- 'content_type' => $rows[0]->content_type,
696		- );
	680	+
	681	+ // DON'T delete the status transients here - let user dismiss manually
697	682	}
698		- }
699	683
700		- // Fallback: fetch by ID
701		- if ( $entry_id > 0 ) {
702		- $row = $wpdb->get_row( $wpdb->prepare(
703		- "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
704		- $entry_id
705		- ) );
706		- if ( $row ) {
707		- $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
708		- return array(
709		- 'content' => $parsed['text'],
710		- 'source_url' => $row->source_url,
711		- 'entry_id' => $row->id,
712		- 'is_chunked' => false,
713		- 'content_type' => $row->content_type,
	684	+ } catch (\Exception $e) {
	685	+ //error_log(sprintf('[MXCHAT-PDF] Error processing PDF: %s', $e->getMessage()));
	686	+
	687	+ $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
	688	+ $status = get_transient($status_key);
	689	+
	690	+ if (!$status || !is_array($status)) {
	691	+ $status = array(
	692	+ 'total_pages' => $total_pages,
	693	+ 'processed_pages' => 0,
	694	+ 'status' => 'error',
	695	+ 'error' => sanitize_text_field($e->getMessage()),
	696	+ 'last_update' => time()
714	697	);
	698	+ } else {
	699	+ $status['status'] = 'error';
	700	+ $status['error'] = sanitize_text_field($e->getMessage());
	701	+ $status['last_update'] = time();
715	702	}
716		- }
717		-
718		- return new WP_Error( 'not_found', 'Entry not found.' );
719		-}
720		-
721		-/**
722		- * Get content from Pinecone — fetches vectors by source_url, reassembles chunks.
723		- */
724		-private function get_pinecone_entry_content( $source_url, $entry_id, $bot_id ) {
725		- if ( ! class_exists('MxChat_Pinecone_Manager') ) {
726		- return new WP_Error( 'pinecone_unavailable', 'Pinecone manager not available.' );
727		- }
728		-
729		- // Get Pinecone config
730		- if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
731		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
732		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
733		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
734		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
735		- } else {
736		- $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
737		- $api_key = $bot_config['api_key'] ?? '';
738		- $host = $bot_config['host'] ?? '';
739		- $namespace = $bot_config['namespace'] ?? '';
740		- }
741		-
742		- if ( empty($host) || empty($api_key) ) {
743		- return new WP_Error( 'pinecone_config', 'Pinecone not configured.' );
744		- }
745		-
746		- // List vectors with the source_url prefix
747		- $base_id = md5( $source_url );
748		- $vector_ids = array( $base_id );
749		-
750		- // Find chunk vectors
751		- $list_url = "https://{$host}/vectors/list";
752		- $list_body = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
753		- if ( ! empty($namespace) ) {
754		- $list_body['namespace'] = $namespace;
755		- }
756		-
757		- $list_resp = wp_remote_post( $list_url, array(
758		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
759		- 'body' => wp_json_encode( $list_body ),
760		- 'timeout' => 15,
761		- ) );
762		-
763		- if ( ! is_wp_error($list_resp) ) {
764		- $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
765		- if ( ! empty($list_data['vectors']) ) {
766		- foreach ( $list_data['vectors'] as $v ) {
767		- $vector_ids[] = $v['id'];
768		- }
	703	+
	704	+ set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
	705	+
	706	+ if (file_exists($pdf_path)) {
	707	+ wp_delete_file($pdf_path);
769	708	}
770	709	}
771		-
772		- // Fetch vectors with metadata
773		- $fetch_url = "https://{$host}/vectors/fetch";
774		- $fetch_body = array( 'ids' => $vector_ids );
775		- if ( ! empty($namespace) ) {
776		- $fetch_body['namespace'] = $namespace;
777		- }
778		-
779		- $fetch_resp = wp_remote_post( $fetch_url, array(
780		- 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
781		- 'body' => wp_json_encode( $fetch_body ),
782		- 'timeout' => 15,
783		- ) );
784		-
785		- if ( is_wp_error($fetch_resp) ) {
786		- return new WP_Error( 'pinecone_fetch', 'Failed to fetch from Pinecone.' );
787		- }
788		-
789		- $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
790		- $vectors = $fetch_data['vectors'] ?? array();
791		-
792		- if ( empty($vectors) ) {
793		- return new WP_Error( 'not_found', 'Entry not found in Pinecone.' );
794		- }
795		-
796		- // Reassemble chunks
797		- $chunks = array();
798		- $content_type = 'content';
799		- foreach ( $vectors as $vid => $vector ) {
800		- $meta = $vector['metadata'] ?? array();
801		- $text = $meta['text'] ?? '';
802		- $index = $meta['chunk_index'] ?? 0;
803		- $content_type = $meta['type'] ?? 'content';
804		- $chunks[ intval($index) ] = $text;
805		- }
806		- ksort( $chunks );
807		-
808		- return array(
809		- 'content' => implode( "\n\n", $chunks ),
810		- 'source_url' => $source_url,
811		- 'is_chunked' => count($chunks) > 1,
812		- 'chunk_count' => count($chunks),
813		- 'content_type' => $content_type,
814		- );
815	710	}
816	711
817		-/**
818		- * AJAX: Save edited content — re-chunks and re-embeds as needed.
819		- * Works for both WordPress DB and Pinecone entries.
820		- */
821		-public function ajax_mxchat_save_entry_content() {
822		- check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
	712	+ public function mxchat_save_inline_prompt() {
	713	+ // Check for nonce security
	714	+ check_ajax_referer('mxchat_save_inline_nonce');
823	715
824		- if ( ! current_user_can('manage_options') ) {
825		- wp_send_json_error( array( 'message' => 'Permission denied.' ) );
826		- }
	716	+ // Verify permissions
	717	+ if (!current_user_can('manage_options')) {
	718	+ wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
	719	+ return;
	720	+ }
827	721
828		- $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
829		- $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
830		- $content = isset($_POST['content']) ? wp_kses_post( wp_unslash($_POST['content']) ) : '';
831		- $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
832		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
833		- $content_type = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : 'content';
	722	+ global $wpdb;
	723	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
834	724
835		- if ( empty($content) ) {
836		- wp_send_json_error( array( 'message' => 'Content cannot be empty.' ) );
837		- }
	725	+ // Validate and sanitize input data
	726	+ $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
	727	+ $article_content = isset($_POST['article_content']) ? sanitize_textarea_field($_POST['article_content']) : '';
	728	+ $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
838	729
839		- // Get the embedding API key
840		- $options = get_option('mxchat_options', array());
841		- $api_key = '';
	730	+ if ($prompt_id > 0 && !empty($article_content)) {
	731	+ // Re-generate the embedding vector for the updated content
	732	+ $embedding_vector = $this->mxchat_generate_embedding($article_content);
842	733
843		- if ( $bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager') ) {
844		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
845		- $api_key = $bot_options['api_key'] ?? '';
846		- }
847		- if ( empty($api_key) ) {
848		- $api_key = $options['api_key'] ?? '';
849		- }
	734	+ if (is_array($embedding_vector)) {
	735	+ // Serialize the embedding vector before storing it
	736	+ $embedding_vector_serialized = serialize($embedding_vector);
850	737
851		- global $wpdb;
852		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
	738	+ // Update the prompt in the database
	739	+ $updated = $wpdb->update(
	740	+ $table_name,
	741	+ array(
	742	+ 'article_content' => $article_content,
	743	+ 'embedding_vector' => $embedding_vector_serialized,
	744	+ 'source_url' => $article_url,
	745	+ ),
	746	+ array('id' => $prompt_id),
	747	+ array('%s', '%s', '%s'),
	748	+ array('%d')
	749	+ );
853	750
854		- // If source_url is empty but we have an entry_id, look it up
855		- if ( empty($source_url) && $entry_id > 0 && $data_source === 'wordpress' ) {
856		- $row = $wpdb->get_row( $wpdb->prepare( "SELECT source_url FROM {$table} WHERE id = %d", $entry_id ) );
857		- if ( $row && ! empty($row->source_url) ) {
858		- $source_url = $row->source_url;
859		- }
860		- }
	751	+ if ($updated !== false) {
	752	+ wp_send_json_success();
	753	+ } else {
	754	+ wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
	755	+ }
	756	+ } else {
	757	+ wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
	758	+ }
	759	+ } else {
	760	+ wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
	761	+ }
	762	+ }
861	763
862		- // For manual entries (no source_url or mxchat:// prefix), delete the old entry by ID first
863		- // so submit_content_to_db creates a replacement instead of a duplicate
864		- // Also treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
865		- $is_legacy_manual = !empty($source_url) && strpos($source_url, 'mxchat.ai') !== false && strpos($source_url, 'mxchat://') !== 0;
866		- if ( $entry_id > 0 && (empty($source_url) || strpos($source_url, 'mxchat://') === 0 || $is_legacy_manual) ) {
867		- $wpdb->delete( $table, array( 'id' => $entry_id ), array( '%d' ) );
868		- // Clear legacy URL so submit_content_to_db generates a unique mxchat:// identifier
869		- // instead of reusing the shared URL (which would mass-delete other entries with the same URL)
870		- if ( $is_legacy_manual ) {
871		- $source_url = '';
872		- }
873		- }
874	764
875		- // Use the existing submit_content_to_db which handles chunking, Pinecone, and WP DB
876		- $vector_id = ! empty($source_url) ? md5($source_url) : md5('mxchat_manual_' . $entry_id);
877		-
878		- // submit_content_to_db already handles: delete old chunks → re-chunk → re-embed → store
879		- $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, $vector_id, $bot_id, $content_type );
880		-
881		- if ( is_wp_error($result) ) {
882		- wp_send_json_error( array( 'message' => $result->get_error_message() ) );
883		- }
884		-
885		- wp_send_json_success( array( 'message' => 'Content saved and re-embedded successfully.' ) );
886		-}
887		-
888	765	public function mxchat_get_pdf_processing_status($pdf_url) {
889	766	$pdf_url = esc_url_raw($pdf_url);
890	767	$status = get_transient(sanitize_key('mxchat_pdf_status_' . md5($pdf_url)));
891	768
		@@ -928,18 +805,24 @@
928	805	}
929	806
930	807
931	808	public function mxchat_handle_sitemap_submission() {
	809	+ // Start logging the submission process
	810	+ //error_log('[MXCHAT-URL] ===== Starting URL submission process =====');
	811	+
932	812	// Check if the form was submitted and verify permissions
933	813	if (!isset($_POST['submit_sitemap']) || !current_user_can('manage_options')) {
	814	+ //error_log('[MXCHAT-URL] Error: Unauthorized access or form not submitted properly');
934	815	wp_die(esc_html__('Unauthorized access', 'mxchat'));
935	816	}
936	817
937	818	// Verify nonce
	819	+ //error_log('[MXCHAT-URL] Verifying nonce');
938	820	check_admin_referer('mxchat_submit_sitemap_action', 'mxchat_submit_sitemap_nonce');
939	821
940	822	// Validate URL
941	823	if (!isset($_POST['sitemap_url']) || empty($_POST['sitemap_url'])) {
	824	+ //error_log('[MXCHAT-URL] Error: Empty or missing URL');
942	825	set_transient('mxchat_admin_notice_error',
943	826	esc_html__('Please provide a valid URL.', 'mxchat'),
944	827	30
945	828	);
		@@ -947,28 +830,12 @@
947	830	exit;
948	831	}
949	832
950	833	$submitted_url = esc_url_raw($_POST['sitemap_url']);
	834	+ //error_log('[MXCHAT-URL] Processing URL: ' . $submitted_url);
951	835
952		- // Convert Google Drive sharing URLs to direct download URLs
953		- if ( strpos($submitted_url, 'drive.google.com') !== false ) {
954		- $file_id = '';
955		- if ( preg_match('/[?&]id=([a-zA-Z0-9_-]+)/', $submitted_url, $m) ) {
956		- $file_id = $m[1];
957		- } elseif ( preg_match('#/file/d/([a-zA-Z0-9_-]+)#', $submitted_url, $m) ) {
958		- $file_id = $m[1];
959		- }
960		- if ( ! empty($file_id) ) {
961		- $submitted_url = 'https://drive.google.com/uc?export=download&id=' . $file_id;
962		- }
963		- }
964		-
965		- // Get bot_id from form submission
966		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
967		-
968		- // Get bot-specific options and validate API key
969		- $bot_options = $this->get_bot_options($bot_id);
970		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	836	+ // Validate API key first
	837	+ $options = get_option('mxchat_options');
971	838	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
972	839
973	840	if (strpos($selected_model, 'voyage') === 0) {
974	841	$api_key = $options['voyage_api_key'] ?? '';
		@@ -985,26 +852,21 @@
985	852	$error_message = sprintf(
986	853	esc_html__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
987	854	$provider_name
988	855	);
	856	+ //error_log('[MXCHAT-URL] Error: ' . $error_message);
989	857	set_transient('mxchat_admin_notice_error', $error_message, 30);
	858	+ //error_log('[MXCHAT-URL] Set error transient: ' . $error_message);
990	859	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
991	860	exit;
992	861	}
993	862
994		- // Fetch URL — use browser-like headers so servers with bot protection don't block us
995		- $response = wp_remote_get($submitted_url, array(
996		- 'timeout' => 30,
997		- 'sslverify' => false,
998		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
999		- 'headers' => array(
1000		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1001		- 'Accept-Language' => 'en-US,en;q=0.9',
1002		- ),
1003		- ));
	863	+ //error_log('[MXCHAT-URL] Fetching URL content');
	864	+ $response = wp_remote_get($submitted_url, array('timeout' => 30));
1004	865
1005	866	if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
1006	867	$error_message = is_wp_error($response) ? $response->get_error_message() : 'HTTP Status: ' . wp_remote_retrieve_response_code($response);
	868	+ //error_log('[MXCHAT-URL] Error fetching URL: ' . $error_message);
1007	869	set_transient('mxchat_admin_notice_error',
1008	870	sprintf(
1009	871	esc_html__('Failed to fetch the URL: %s', 'mxchat'),
1010	872	esc_html($error_message)
		@@ -1015,11 +877,13 @@
1015	877	exit;
1016	878	}
1017	879
1018	880	$content_type = wp_remote_retrieve_header($response, 'content-type');
	881	+ //error_log('[MXCHAT-URL] Content type: ' . $content_type);
1019	882	$body_content = wp_remote_retrieve_body($response);
1020	883
1021	884	if (empty($body_content)) {
	885	+ //error_log('[MXCHAT-URL] Error: Empty response body');
1022	886	set_transient('mxchat_admin_notice_error',
1023	887	esc_html__('Empty response received from URL.', 'mxchat'),
1024	888	30
1025	889	);
		@@ -1025,21 +889,29 @@
1025	889	);
1026	890	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1027	891	exit;
1028	892	}
	893	+ //error_log('[MXCHAT-URL] Retrieved body content length: ' . strlen($body_content) . ' bytes');
1029	894
1030	895	// Handle PDF URL
1031	896	if ($this->mxchat_is_pdf_url($submitted_url, $response)) {
1032		- $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response, $bot_id);
	897	+ //error_log('[MXCHAT-URL] Detected PDF URL, handling PDF for knowledge base');
	898	+ $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response);
	899	+ //error_log('[MXCHAT-URL] PDF handling result: ' . $result);
1033	900
1034		- if ($result === 'queued') {
1035		- set_transient('mxchat_admin_notice_success',
1036		- esc_html__('PDF queued for processing. Processing will start automatically.', 'mxchat'),
	901	+ if ($result === 'scheduled') {
	902	+ set_transient(
	903	+ 'mxchat_last_pdf_url',
	904	+ sanitize_text_field($submitted_url),
	905	+ DAY_IN_SECONDS
	906	+ );
	907	+ set_transient('mxchat_admin_notice_info',
	908	+ esc_html__('PDF processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1037	909	30
1038	910	);
1039	911	} else {
1040	912	set_transient('mxchat_admin_notice_error',
1041		- esc_html__('Failed to queue PDF processing: ', 'mxchat') . esc_html($result),
	913	+ esc_html__('Failed to start PDF processing: ', 'mxchat') . esc_html($result),
1042	914	30
1043	915	);
1044	916	}
1045	917
		@@ -1048,8 +920,9 @@
1048	920	}
1049	921
1050	922	// Handle Sitemap XML
1051	923	if (strpos($content_type, 'xml') !== false || strpos($body_content, '<urlset') !== false) {
	924	+ //error_log('[MXCHAT-URL] Detected XML content, processing as sitemap');
1052	925	libxml_use_internal_errors(true);
1053	926	$xml = simplexml_load_string($body_content);
1054	927	$xml_errors = libxml_get_errors();
1055	928	libxml_clear_errors();
		@@ -1054,8 +927,15 @@
1054	927	$xml_errors = libxml_get_errors();
1055	928	libxml_clear_errors();
1056	929
1057	930	if ($xml === false || !empty($xml_errors)) {
	931	+ //error_log('[MXCHAT-URL] Error: Invalid XML format');
	932	+ if (!empty($xml_errors)) {
	933	+ foreach ($xml_errors as $error) {
	934	+ //error_log('[MXCHAT-URL] XML Error: ' . $error->message);
	935	+ }
	936	+ }
	937	+
1058	938	set_transient('mxchat_admin_notice_error',
1059	939	esc_html__('Invalid sitemap XML. Please provide a valid sitemap.', 'mxchat'),
1060	940	30
1061	941	);
		@@ -1062,18 +942,27 @@
1062	942	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1063	943	exit;
1064	944	}
1065	945
1066		- $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url, $bot_id);
	946	+ //error_log('[MXCHAT-URL] Valid XML found, handling sitemap for knowledge base');
	947	+ $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url);
	948	+ //error_log('[MXCHAT-URL] Sitemap handling result: ' . $result);
1067	949
1068		- if ($result === 'queued') {
1069		- set_transient('mxchat_admin_notice_success',
1070		- esc_html__('Sitemap queued for processing. Processing will start automatically.', 'mxchat'),
	950	+ if ($result === 'scheduled') {
	951	+ set_transient(
	952	+ 'mxchat_last_sitemap_url',
	953	+ sanitize_text_field($submitted_url),
	954	+ DAY_IN_SECONDS
	955	+ );
	956	+ set_transient('mxchat_admin_notice_info',
	957	+ esc_html__('Sitemap processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1071	958	30
1072	959	);
1073	960	} else {
	961	+ // Return to the admin page without a redirect for better error display
	962	+ // The error is already stored in the sitemap status transient
1074	963	set_transient('mxchat_admin_notice_error',
1075		- esc_html__('Failed to queue sitemap processing. Please check the status below for details.', 'mxchat'),
	964	+ esc_html__('Failed to start sitemap processing. Please check the status below for details.', 'mxchat'),
1076	965	30
1077	966	);
1078	967	}
1079	968
		@@ -1080,50 +969,127 @@
1080	969	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1081	970	exit;
1082	971	}
1083	972
1084		- // Handle Regular URL (single page)
	973	+ // Handle Regular URL
	974	+ //error_log('[MXCHAT-URL] Processing as regular webpage');
1085	975	$page_content = $this->mxchat_extract_main_content($body_content);
	976	+ //error_log('[MXCHAT-URL] Extracted content length: ' . strlen($page_content) . ' bytes');
	977	+
1086	978	$sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	979	+ //error_log('[MXCHAT-URL] Sanitized content length: ' . strlen($sanitized_content) . ' bytes');
1087	980
1088		- //error_log('[MXCHAT-URL-DEBUG] URL: ' . $submitted_url);
1089		- //error_log('[MXCHAT-URL-DEBUG] Extracted page_content length: ' . strlen($page_content));
1090		- //error_log('[MXCHAT-URL-DEBUG] Sanitized content length: ' . strlen($sanitized_content));
1091		- //error_log('[MXCHAT-URL-DEBUG] Content preview: ' . substr($sanitized_content, 0, 500));
1092		-
1093	981	if (empty($sanitized_content)) {
	982	+ //error_log('[MXCHAT-URL] Error: No valid content after sanitization');
	983	+
	984	+ // Set both transients - the error notice and the URL status
1094	985	set_transient('mxchat_admin_notice_error',
1095	986	esc_html__('No valid content found on the provided URL.', 'mxchat'),
1096	987	30
1097	988	);
	989	+
	990	+ // Set URL status transient
	991	+ set_transient('mxchat_single_url_status', [
	992	+ 'url' => $submitted_url,
	993	+ 'timestamp' => current_time('mysql'),
	994	+ 'status' => 'failed',
	995	+ 'error' => esc_html__('No valid content found on the provided URL.', 'mxchat')
	996	+ ], DAY_IN_SECONDS);
	997	+
1098	998	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1099	999	exit;
1100	1000	}
1101	1001
1102		- // For single URLs, process immediately using submit_content_to_db
1103		- // This handles chunking automatically for large content
1104		- $db_result = MxChat_Utils::submit_content_to_db(
1105		- $sanitized_content,
1106		- $submitted_url,
1107		- $api_key,
1108		- null,
1109		- $bot_id,
1110		- 'url' // content_type
1111		- );
	1002	+ //error_log('[MXCHAT-URL] Generating embedding for content');
	1003	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
1112	1004
1113		- if (is_wp_error($db_result)) {
1114		- $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1005	+ // Check if embedding_vector is a string (error message)
	1006	+ if (is_string($embedding_vector)) {
	1007	+ //error_log('[MXCHAT-URL] Error generating embedding: ' . $embedding_vector);
	1008	+ $error_message = esc_html__('Failed to generate embedding: ', 'mxchat') . esc_html($embedding_vector);
	1009	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1010	+
	1011	+ // Set both transients
1115	1012	set_transient('mxchat_admin_notice_error', $error_message, 30);
1116		- } else {
	1013	+
	1014	+ // Set URL status transient
	1015	+ set_transient('mxchat_single_url_status', [
	1016	+ 'url' => $submitted_url,
	1017	+ 'timestamp' => current_time('mysql'),
	1018	+ 'status' => 'failed',
	1019	+ 'error' => $error_message
	1020	+ ], DAY_IN_SECONDS);
	1021	+
	1022	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1023	+ exit;
	1024	+ }
	1025	+
	1026	+ if (is_array($embedding_vector)) {
	1027	+ //error_log('[MXCHAT-URL] Successfully generated embedding with ' . count($embedding_vector) . ' dimensions');
	1028	+
	1029	+ $db_result = MxChat_Utils::submit_content_to_db(
	1030	+ $sanitized_content,
	1031	+ $submitted_url,
	1032	+ $api_key
	1033	+ );
	1034	+
	1035	+ if (is_wp_error($db_result)) {
	1036	+ //error_log('[MXCHAT-URL] Error: Failed to store content in database: ' . $db_result->get_error_message());
	1037	+ $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
	1038	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1039	+
	1040	+ // Set both transients
	1041	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1042	+
	1043	+ // Set URL status transient
	1044	+ set_transient('mxchat_single_url_status', [
	1045	+ 'url' => $submitted_url,
	1046	+ 'timestamp' => current_time('mysql'),
	1047	+ 'status' => 'failed',
	1048	+ 'error' => $error_message
	1049	+ ], DAY_IN_SECONDS);
	1050	+
	1051	+ wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
	1052	+ exit;
	1053	+ }
	1054	+
	1055	+ //error_log('[MXCHAT-URL] Successfully stored content in database');
1117	1056	$success_message = esc_html__('URL content successfully submitted!', 'mxchat');
	1057	+ //error_log('[MXCHAT-URL] Setting success transient: ' . $success_message);
	1058	+
	1059	+ // Set both transients
1118	1060	set_transient('mxchat_admin_notice_success', $success_message, 30);
	1061	+
	1062	+ // Set URL status transient with success
	1063	+ set_transient('mxchat_single_url_status', [
	1064	+ 'url' => $submitted_url,
	1065	+ 'timestamp' => current_time('mysql'),
	1066	+ 'status' => 'complete',
	1067	+ 'content_length' => strlen($sanitized_content),
	1068	+ 'embedding_dimensions' => count($embedding_vector)
	1069	+ ], DAY_IN_SECONDS);
	1070	+
	1071	+ } else {
	1072	+ //error_log('[MXCHAT-URL] Error: Failed to generate embedding. Unexpected result type: ' . gettype($embedding_vector));
	1073	+ $error_message = esc_html__('Failed to generate embedding: Unexpected result type. Please check your API key and try again.', 'mxchat');
	1074	+ //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
	1075	+
	1076	+ // Set both transients
	1077	+ set_transient('mxchat_admin_notice_error', $error_message, 30);
	1078	+
	1079	+ // Set URL status transient
	1080	+ set_transient('mxchat_single_url_status', [
	1081	+ 'url' => $submitted_url,
	1082	+ 'timestamp' => current_time('mysql'),
	1083	+ 'status' => 'failed',
	1084	+ 'error' => $error_message
	1085	+ ], DAY_IN_SECONDS);
1119	1086	}
1120	1087
	1088	+ //error_log('[MXCHAT-URL] ===== Completed URL submission process =====');
1121	1089	wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1122	1090	exit;
1123	1091	}
1124		-
1125		-
1126	1092	public function mxchat_get_single_url_status() {
1127	1093	$status = get_transient('mxchat_single_url_status');
1128	1094	if (!$status) {
1129	1095	return null;
		@@ -1135,11 +1101,13 @@
1135	1101	}
1136	1102
1137	1103	return $status;
1138	1104	}
1139		-
1140		-public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url, $bot_id = 'default') {
	1105	+public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url) {
	1106	+ // Clear any single URL status when starting sitemap processing
	1107	+ delete_transient('mxchat_single_url_status');
1141	1108	if (!current_user_can('manage_options')) {
	1109	+ //error_log(esc_html__('Unauthorized sitemap processing attempt', 'mxchat'));
1142	1110	return false;
1143	1111	}
1144	1112
1145	1113	try {
		@@ -1148,96 +1116,331 @@
1148	1116	if (!$xml || !is_object($xml)) {
1149	1117	throw new Exception(__('Invalid XML object provided', 'mxchat'));
1150	1118	}
1151	1119
1152		- // Get bot-specific embedding API for validation
1153		- $bot_options = $this->get_bot_options($bot_id);
1154		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	1120	+ // Add embedding validation before processing
	1121	+ // Test embedding with a small sample text to verify API key is working
	1122	+ $test_result = $this->mxchat_generate_embedding("This is a test to verify the embedding API key is working.");
1155	1123
1156		- // Test the embedding API before processing
1157		- $test_phrase = "Test embedding generation for MxChat";
1158		- $test_result = $this->mxchat_generate_embedding($test_phrase, $bot_id);
1159		-
	1124	+ // Check if test_result is a string (error message) rather than an array (valid embedding)
1160	1125	if (is_string($test_result)) {
	1126	+ //error_log('[MXCHAT-URL] Embedding API validation failed: ' . $test_result);
	1127	+
	1128	+ // Store the error in the status transient so it can be displayed later
	1129	+ $status_data = array(
	1130	+ 'total_urls' => 0,
	1131	+ 'processed_urls' => 0,
	1132	+ 'status' => 'error',
	1133	+ 'error' => __('Embedding API validation failed: ', 'mxchat') . $test_result,
	1134	+ 'last_update' => time()
	1135	+ );
	1136	+
	1137	+ set_transient(
	1138	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1139	+ array_map('sanitize_text_field', $status_data),
	1140	+ DAY_IN_SECONDS
	1141	+ );
	1142	+
1161	1143	throw new Exception(__('Embedding API validation failed: ', 'mxchat') . $test_result);
1162	1144	}
1163	1145
	1146	+ // Make sure it's an array (valid embedding)
1164	1147	if (!is_array($test_result)) {
	1148	+ //error_log('[MXCHAT-URL] Embedding API returned unexpected result type: ' . gettype($test_result));
1165	1149	throw new Exception(__('Embedding API returned unexpected result type. Please check your configuration.', 'mxchat'));
1166	1150	}
1167	1151
1168		- // Extract URLs from sitemap
1169		- $urls = array();
	1152	+ $urls = [];
1170	1153	foreach ($xml->url as $url_element) {
1171	1154	$url = esc_url_raw((string)$url_element->loc);
1172	1155	if ($url) {
1173		- $urls[] = array('url' => $url);
	1156	+ $urls[] = $url;
1174	1157	}
1175	1158	}
1176	1159
1177		- $total_urls = count($urls);
	1160	+ $total_urls = absint(count($urls));
1178	1161
1179	1162	if ($total_urls < 1) {
1180	1163	throw new Exception(__('No valid URLs found in sitemap', 'mxchat'));
1181	1164	}
1182	1165
1183		- // Create unique queue ID
1184		- $queue_id = 'sitemap_' . md5($sitemap_url . time());
	1166	+ wp_schedule_single_event(time(), 'mxchat_process_sitemap_urls', array(
	1167	+ 'urls' => $urls,
	1168	+ 'sitemap_url' => $sitemap_url,
	1169	+ 'total_urls' => $total_urls,
	1170	+ 'batch_size' => absint(10),
	1171	+ 'batch_pause' => absint(5)
	1172	+ ));
1185	1173
1186		- // Add URLs to queue
1187		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'url', $urls, $bot_id);
	1174	+ $status_data = array(
	1175	+ 'total_urls' => $total_urls,
	1176	+ 'processed_urls' => 0,
	1177	+ 'status' => 'processing',
	1178	+ 'last_update' => time()
	1179	+ );
1188	1180
1189		- if ($queued_count === 0) {
1190		- throw new Exception(__('Failed to add URLs to processing queue', 'mxchat'));
1191		- }
	1181	+ set_transient(
	1182	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1183	+ array_map('sanitize_text_field', $status_data),
	1184	+ DAY_IN_SECONDS
	1185	+ );
1192	1186
1193		- // Store queue metadata
1194		- $this->mxchat_set_queue_meta($queue_id, 'source_url', $sitemap_url);
1195		- $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'sitemap');
1196		- $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_urls);
1197		- $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
1198		- $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
	1187	+ return __('scheduled', 'mxchat');
1199	1188
1200		- // Store queue ID in transient for status tracking
1201		- set_transient('mxchat_active_queue_sitemap', $queue_id, DAY_IN_SECONDS);
1202		- set_transient('mxchat_last_sitemap_url', $sitemap_url, DAY_IN_SECONDS);
1203		-
1204		- return 'queued';
1205		-
1206		- } catch (Exception $e) {
	1189	+ } catch (\Exception $e) {
1207	1190	$error_message = $e->getMessage();
1208	1191	//error_log(sprintf(esc_html__('Error preparing sitemap for processing: %s', 'mxchat'), esc_html($error_message)));
1209	1192
	1193	+ // Store the sitemap URL and error in transients so they can be displayed
	1194	+ set_transient(
	1195	+ 'mxchat_last_sitemap_url',
	1196	+ sanitize_text_field($sitemap_url),
	1197	+ DAY_IN_SECONDS
	1198	+ );
	1199	+
	1200	+ $status_data = array(
	1201	+ 'total_urls' => 0,
	1202	+ 'processed_urls' => 0,
	1203	+ 'status' => 'error',
	1204	+ 'error' => $error_message,
	1205	+ 'last_update' => time()
	1206	+ );
	1207	+
	1208	+ set_transient(
	1209	+ sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
	1210	+ array_map('sanitize_text_field', $status_data),
	1211	+ DAY_IN_SECONDS
	1212	+ );
	1213	+
1210	1214	return $error_message;
1211	1215	}
1212		-
1213	1216	}
1214	1217
1215		-/**
1216		- * Remove shortcode tags but preserve the content inside them
1217		- * Example: [vc_column]Hello World[/vc_column] becomes "Hello World"
1218		- *
1219		- * @param string $content The content containing shortcodes
1220		- * @return string Content with shortcode tags removed but inner content preserved
1221		- */
1222		-private function strip_shortcode_tags_preserve_content($content) {
1223		- // Single-pass regex removes all shortcode brackets: [tag], [tag attr="val"], [/tag], [tag /]
1224		- // Content between tags is inherently preserved since only brackets are targeted
1225		- $result = preg_replace('/\[\/?\w[\w-]*[^\]]*\]/', '', $content);
1226		- return ($result !== null) ? $result : $content;
	1218	+public function mxchat_process_sitemap_urls_cron($urls, $sitemap_url, $total_urls, $batch_size, $batch_pause) {
	1219	+ // Validate inputs
	1220	+ $sitemap_url = esc_url_raw($sitemap_url);
	1221	+ $total_urls = absint($total_urls);
	1222	+ $batch_size = absint($batch_size);
	1223	+ $batch_pause = absint($batch_pause);
	1224	+
	1225	+ if (!is_array($urls) || empty($urls)) {
	1226	+ return;
	1227	+ }
	1228	+
	1229	+ try {
	1230	+ $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
	1231	+ $status = get_transient($status_key);
	1232	+
	1233	+ if (!$status || !is_array($status)) {
	1234	+ throw new Exception('Invalid status data retrieved from transient');
	1235	+ }
	1236	+
	1237	+ // Initialize failed_urls array if it doesn't exist
	1238	+ if (!isset($status['failed_urls_list']) || !is_array($status['failed_urls_list'])) {
	1239	+ $status['failed_urls_list'] = [];
	1240	+ }
	1241	+
	1242	+ $start_url = absint($status['processed_urls']);
	1243	+ $end_url = min($start_url + $batch_size, $total_urls);
	1244	+
	1245	+ // Track batch statistics
	1246	+ $batch_stats = [
	1247	+ 'processed' => 0,
	1248	+ 'failed' => 0,
	1249	+ 'last_error' => '',
	1250	+ 'embedding_errors' => 0
	1251	+ ];
	1252	+
	1253	+ // Check embedding configuration with first URL (only on first batch)
	1254	+ if ($start_url === 0) {
	1255	+ $test_url = esc_url_raw($urls[0]);
	1256	+ $test_response = wp_remote_get($test_url);
	1257	+
	1258	+ if (!is_wp_error($test_response) && wp_remote_retrieve_response_code($test_response) === 200) {
	1259	+ $test_html = wp_remote_retrieve_body($test_response);
	1260	+ $test_content = $this->mxchat_extract_main_content($test_html);
	1261	+ $test_sanitized = $this->mxchat_sanitize_content_for_api($test_content);
	1262	+
	1263	+ if (!empty($test_sanitized)) {
	1264	+ $test_embedding = $this->mxchat_generate_embedding($test_sanitized);
	1265	+
	1266	+ if (is_string($test_embedding)) {
	1267	+ throw new Exception('Embedding generation failed: ' . $test_embedding);
	1268	+ }
	1269	+
	1270	+ if (!is_array($test_embedding)) {
	1271	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($test_embedding));
	1272	+ }
	1273	+ }
	1274	+ }
	1275	+ }
	1276	+
	1277	+ for ($i = $start_url; $i < $end_url; $i++) {
	1278	+ $page_url = esc_url_raw($urls[$i]);
	1279	+ $max_retries = 3;
	1280	+ $retry_count = 0;
	1281	+ $url_processed = false;
	1282	+ $last_error = '';
	1283	+
	1284	+ while (!$url_processed && $retry_count < $max_retries) {
	1285	+ try {
	1286	+ // Attempt to fetch the URL
	1287	+ $page_response = wp_remote_get($page_url, array('timeout' => 30));
	1288	+
	1289	+ if (is_wp_error($page_response)) {
	1290	+ throw new Exception('HTTP request failed: ' . $page_response->get_error_message());
	1291	+ }
	1292	+
	1293	+ $response_code = wp_remote_retrieve_response_code($page_response);
	1294	+ if ($response_code !== 200) {
	1295	+ throw new Exception('HTTP Status: ' . $response_code);
	1296	+ }
	1297	+
	1298	+ $page_html = wp_remote_retrieve_body($page_response);
	1299	+
	1300	+ if (empty($page_html)) {
	1301	+ throw new Exception('Empty response body');
	1302	+ }
	1303	+
	1304	+ $page_content = $this->mxchat_extract_main_content($page_html);
	1305	+ $sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
	1306	+
	1307	+ if (empty($sanitized_content)) {
	1308	+ throw new Exception('No valid content found after processing');
	1309	+ }
	1310	+
	1311	+ $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
	1312	+
	1313	+ if (is_string($embedding_vector)) {
	1314	+ throw new Exception('Embedding generation failed: ' . $embedding_vector);
	1315	+ }
	1316	+
	1317	+ if (!is_array($embedding_vector)) {
	1318	+ throw new Exception('Embedding generation returned unexpected result type: ' . gettype($embedding_vector));
	1319	+ }
	1320	+
	1321	+ // Submit to database
	1322	+ $options = get_option('mxchat_options');
	1323	+ $submission_result = MxChat_Utils::submit_content_to_db($sanitized_content, $page_url, $options['api_key']);
	1324	+
	1325	+ if (is_wp_error($submission_result)) {
	1326	+ throw new Exception('Database submission failed: ' . $submission_result->get_error_message());
	1327	+ }
	1328	+
	1329	+ // Success!
	1330	+ $url_processed = true;
	1331	+ $batch_stats['processed']++;
	1332	+
	1333	+ } catch (Exception $e) {
	1334	+ $retry_count++;
	1335	+ $last_error = $e->getMessage();
	1336	+
	1337	+ //error_log("URL {$page_url} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
	1338	+
	1339	+ // Track embedding errors specifically
	1340	+ if (strpos($last_error, 'Embedding') !== false) {
	1341	+ $batch_stats['embedding_errors']++;
	1342	+ }
	1343	+
	1344	+ if ($retry_count < $max_retries) {
	1345	+ // Wait before retry (exponential backoff: 1s, 2s, 4s)
	1346	+ sleep(pow(2, $retry_count - 1));
	1347	+ }
	1348	+ }
	1349	+ }
	1350	+
	1351	+ // If URL still not processed after all retries, mark as failed
	1352	+ if (!$url_processed) {
	1353	+ $batch_stats['failed']++;
	1354	+ $batch_stats['last_error'] = $last_error;
	1355	+
	1356	+ // Add to failed URLs list
	1357	+ $status['failed_urls_list'][] = [
	1358	+ 'url' => $page_url,
	1359	+ 'error' => $last_error,
	1360	+ 'time' => time(),
	1361	+ 'retries' => $max_retries
	1362	+ ];
	1363	+
	1364	+ // Limit the number of failed URLs we store to prevent transient size issues
	1365	+ if (count($status['failed_urls_list']) > 100) {
	1366	+ $status['failed_urls_list'] = array_slice($status['failed_urls_list'], -100);
	1367	+ }
	1368	+ }
	1369	+
	1370	+ // Update progress
	1371	+ $status['processed_urls'] = absint($i + 1);
	1372	+ $status['last_update'] = time();
	1373	+ $status['failed_urls'] = absint($status['failed_urls'] ?? 0) + ($url_processed ? 0 : 1);
	1374	+ $status['last_error'] = $batch_stats['last_error'];
	1375	+
	1376	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1377	+
	1378	+ // If we have too many consecutive embedding errors, stop processing
	1379	+ if ($batch_stats['embedding_errors'] >= 10) {
	1380	+ throw new Exception('Too many consecutive embedding failures detected. Please check your API configuration.');
	1381	+ }
	1382	+ }
	1383	+
	1384	+ // If all URLs in this batch failed, stop processing
	1385	+ if ($batch_stats['processed'] === 0 && $batch_stats['failed'] > 0) {
	1386	+ $status['status'] = 'error';
	1387	+ $status['error'] = sprintf(
	1388	+ 'Processing stopped: %d consecutive failures in batch. Last error: %s',
	1389	+ $batch_stats['failed'],
	1390	+ $batch_stats['last_error']
	1391	+ );
	1392	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1393	+ return;
	1394	+ }
	1395	+
	1396	+ // Update final progress
	1397	+ $status['processed_urls'] = min($end_url, $total_urls);
	1398	+ $status['last_update'] = time();
	1399	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1400	+
	1401	+ // Check if we've processed all URLs
	1402	+ if ($end_url >= $total_urls) {
	1403	+ // All URLs have been processed - mark as complete
	1404	+ $status['status'] = 'complete';
	1405	+ $status['processed_urls'] = $total_urls;
	1406	+
	1407	+ // Add completion summary
	1408	+ $status['completion_summary'] = [
	1409	+ 'total_urls' => $total_urls,
	1410	+ 'successful_urls' => $total_urls - absint($status['failed_urls'] ?? 0),
	1411	+ 'failed_urls' => absint($status['failed_urls'] ?? 0),
	1412	+ 'completion_time' => current_time('mysql')
	1413	+ ];
	1414	+
	1415	+ // Save the completed status (don't delete it - let user dismiss manually)
	1416	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1417	+
	1418	+ // DON'T delete the status transients here - let user dismiss manually
	1419	+ } else {
	1420	+ // Schedule next batch
	1421	+ wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_sitemap_urls', array(
	1422	+ 'urls' => $urls,
	1423	+ 'sitemap_url' => $sitemap_url,
	1424	+ 'total_urls' => $total_urls,
	1425	+ 'batch_size' => $batch_size,
	1426	+ 'batch_pause' => $batch_pause,
	1427	+ ));
	1428	+ }
	1429	+ } catch (\Exception $e) {
	1430	+ $status['status'] = 'error';
	1431	+ $status['error'] = $e->getMessage();
	1432	+ set_transient($status_key, $status, DAY_IN_SECONDS);
	1433	+ }
1227	1434	}
1228		-
1229	1435	public function mxchat_sanitize_content_for_api($content) {
1230	1436	//error_log('[MXCHAT-SANITIZE] Original content preview: ' . substr($content, 0, 500) . '...');
1231		-
1232		- // Remove shortcode tags but PRESERVE content inside them
1233		- $content = $this->strip_shortcode_tags_preserve_content($content);
1234		-
	1437	+
1235	1438	// Remove script, style tags, and HTML comments
1236	1439	$content = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $content);
1237	1440	$content = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $content);
1238	1441	$content = preg_replace('/<!--(.|\s)*?-->/', '', $content);
1239		-
	1442	+
1240	1443	// Remove all HTML tags and decode HTML entities
1241	1444	$content = wp_strip_all_tags($content);
1242	1445	$content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5);
1243	1446
		@@ -1295,12 +1498,12 @@
1295	1498	@$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'), LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
1296	1499	$xpath = new DOMXPath($dom);
1297	1500
1298	1501	// For debugging purposes
1299		- $debugEnabled = true; // Set to true to enable debugging output
	1502	+ $debugEnabled = false; // Set to true to enable debugging output
1300	1503	$debug = function($message) use ($debugEnabled) {
1301	1504	if ($debugEnabled) {
1302		- //error_log('[MXCHAT-EXTRACT-DEBUG] ' . $message);
	1505	+ //error_log('[MXCHAT-DEBUG] ' . $message);
1303	1506	}
1304	1507	};
1305	1508
1306	1509	// Direct targeting for Gerow theme posts
		@@ -1399,25 +1602,18 @@
1399	1602	'//main',
1400	1603	'//div[contains(@class, "content")]'
1401	1604	];
1402	1605
1403		- // First handle Elementor content - get only leaf widget containers to avoid duplicates
	1606	+ // First handle Elementor content
1404	1607	$debug("Checking for Elementor content");
1405		- // Get widget containers that are direct children of widgets (not nested inside other widget containers)
1406		- $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-widget")]//div[contains(@class, "elementor-widget-container")]');
	1608	+ $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-element")]//div[contains(@class, "elementor-widget-container")]');
1407	1609	if ($elementor_widgets && $elementor_widgets->length > 0) {
1408	1610	$debug("Found Elementor widgets");
1409		- $seen_content = array(); // Track seen content to avoid duplicates
1410	1611	$combined_content = '';
1411	1612	foreach ($elementor_widgets as $widget) {
1412	1613	$widget_content = $dom->saveHTML($widget);
1413	1614	if (!empty($widget_content)) {
1414		- // Create a hash of the content to detect duplicates
1415		- $content_hash = md5($widget_content);
1416		- if (!isset($seen_content[$content_hash])) {
1417		- $seen_content[$content_hash] = true;
1418		- $combined_content .= $widget_content;
1419		- }
	1615	+ $combined_content .= $widget_content;
1420	1616	}
1421	1617	}
1422	1618	if (!empty($combined_content)) {
1423	1619	$debug("Returning Elementor content");
		@@ -1429,14 +1625,15 @@
1429	1625	foreach ($selectors as $selector) {
1430	1626	$debug("Trying selector: " . $selector);
1431	1627	$nodes = $xpath->query($selector);
1432	1628	if ($nodes && $nodes->length > 0) {
1433		- $debug("Found " . $nodes->length . " matches for selector: " . $selector);
1434		- // Only take the FIRST matching node to avoid duplicate content
1435		- // (pages often have nested or multiple containers with same class)
1436		- $content = $dom->saveHTML($nodes->item(0));
	1629	+ $debug("Found matches for selector: " . $selector);
	1630	+ $content = '';
	1631	+ foreach ($nodes as $node) {
	1632	+ $content .= $dom->saveHTML($node);
	1633	+ }
1437	1634	if (!empty($content)) {
1438		- $debug("Returning content from selector: " . $selector . " (first match only)");
	1635	+ $debug("Returning content from selector: " . $selector);
1439	1636	return $content;
1440	1637	}
1441	1638	}
1442	1639	}
		@@ -1460,106 +1657,16 @@
1460	1657	$debug("Returning blog-area section content");
1461	1658	return $content;
1462	1659	}
1463	1660	}
1464		-
1465		- // Generic container selectors for non-CMS sites (like .asp pages)
1466		- $debug("Trying generic container selectors");
1467		- $generic_selectors = [
1468		- '//div[@id="main"]',
1469		- '//div[@id="wrapper"]',
1470		- '//div[@id="page"]',
1471		- '//div[@id="site-content"]',
1472		- '//div[contains(@class, "main-content")]',
1473		- '//div[contains(@class, "page-content")]',
1474		- '//div[contains(@class, "site-content")]',
1475		- ];
1476		-
1477		- foreach ($generic_selectors as $selector) {
1478		- $debug("Trying generic selector: " . $selector);
1479		- $nodes = $xpath->query($selector);
1480		- if ($nodes && $nodes->length > 0) {
1481		- $content = $dom->saveHTML($nodes->item(0));
1482		- if (!empty($content)) {
1483		- $debug("Returning content from generic selector: " . $selector);
1484		- return $content;
1485		- }
1486		- }
1487		- }
1488		-
1489		- // Paragraph-based content detection - find regions with substantial text
1490		- $debug("Trying paragraph-based content detection");
1491		- $paragraphs = $xpath->query('//p[string-length(normalize-space()) > 50]');
1492		- if ($paragraphs && $paragraphs->length >= 3) {
1493		- $debug("Found " . $paragraphs->length . " substantial paragraphs");
1494		- // Collect all substantial paragraphs and their content
1495		- $paragraph_content = '';
1496		- foreach ($paragraphs as $p) {
1497		- $paragraph_content .= $dom->saveHTML($p) . "\n";
1498		- }
1499		- if (!empty($paragraph_content)) {
1500		- $debug("Returning paragraph-based content");
1501		- return $paragraph_content;
1502		- }
1503		- }
1504		-
1505		- // Improved body fallback - strip nav/header/footer elements first
1506		- $debug("Using improved body fallback");
	1661	+
	1662	+ // Fallback: Return the body content if no specific selector matches
	1663	+ $debug("Using body fallback");
1507	1664	$body = $dom->getElementsByTagName('body');
1508	1665	if ($body->length > 0) {
1509		- // Clone the body to avoid modifying the original DOM
1510		- $body_clone = $body->item(0)->cloneNode(true);
1511		-
1512		- // Remove common non-content elements by tag name
1513		- $remove_tags = ['nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript'];
1514		- foreach ($remove_tags as $tag) {
1515		- $elements = $body_clone->getElementsByTagName($tag);
1516		- // Iterate backwards to safely remove elements
1517		- for ($i = $elements->length - 1; $i >= 0; $i--) {
1518		- $el = $elements->item($i);
1519		- if ($el && $el->parentNode) {
1520		- $el->parentNode->removeChild($el);
1521		- }
1522		- }
1523		- }
1524		-
1525		- // Remove elements with common non-content class names using XPath on the cloned body
1526		- $temp_dom = new DOMDocument();
1527		- @$temp_dom->appendChild($temp_dom->importNode($body_clone, true));
1528		- $temp_xpath = new DOMXPath($temp_dom);
1529		-
1530		- $remove_class_patterns = [
1531		- '//*[contains(@class, "nav")]',
1532		- '//*[contains(@class, "menu")]',
1533		- '//*[contains(@class, "sidebar")]',
1534		- '//*[contains(@class, "footer")]',
1535		- '//*[contains(@class, "header")]',
1536		- '//*[contains(@id, "nav")]',
1537		- '//*[contains(@id, "menu")]',
1538		- '//*[contains(@id, "sidebar")]',
1539		- '//*[contains(@id, "footer")]',
1540		- '//*[contains(@id, "header")]',
1541		- ];
1542		-
1543		- foreach ($remove_class_patterns as $pattern) {
1544		- $elements = $temp_xpath->query($pattern);
1545		- if ($elements) {
1546		- for ($i = $elements->length - 1; $i >= 0; $i--) {
1547		- $el = $elements->item($i);
1548		- if ($el && $el->parentNode) {
1549		- $el->parentNode->removeChild($el);
1550		- }
1551		- }
1552		- }
1553		- }
1554		-
1555		- $cleaned_content = $temp_dom->saveHTML();
1556		- if (!empty($cleaned_content)) {
1557		- $debug("Returning cleaned body content");
1558		- return $cleaned_content;
1559		- }
	1666	+ return $dom->saveHTML($body->item(0));
1560	1667	}
1561		-
	1668	+
1562	1669	// Last resort: return the original HTML
1563	1670	$debug("Returning original HTML");
1564	1671	return $html;
1565	1672	} catch (Exception $e) {
		@@ -1611,41 +1718,47 @@
1611	1718	try {
1612	1719	// Verify the request
1613	1720	check_ajax_referer('mxchat_status_nonce', 'nonce');
1614	1721
1615		- // Get active queue IDs
1616		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
1617		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
	1722	+ // Get the status just like in your admin page
	1723	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1724	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
1618	1725
1619		- $sitemap_status = false;
1620		- $pdf_status = false;
	1726	+ $pdf_status = $pdf_url ? $this->mxchat_get_pdf_processing_status($pdf_url) : false;
	1727	+ $sitemap_status = $sitemap_url ? $this->mxchat_get_sitemap_processing_status($sitemap_url) : false;
1621	1728
1622		- // Get sitemap queue status
1623		- if ($sitemap_queue_id) {
1624		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
	1729	+ // Add the PDF URL to the status object
	1730	+ if ($pdf_status && $pdf_url) {
	1731	+ $pdf_status['pdf_url'] = $pdf_url;
1625	1732	}
1626	1733
1627		- // Get PDF queue status
1628		- if ($pdf_queue_id) {
1629		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
1630		- }
	1734	+ // Set the current PDF URL for the manual batch processing button
	1735	+ $current_pdf_url = $pdf_url;
1631	1736
	1737	+ // Check for true processing status, not just presence of status
1632	1738	$is_active_processing =
1633		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
1634		- ($pdf_status && $pdf_status['status'] === 'processing');
	1739	+ ($sitemap_status && isset($sitemap_status['status']) && $sitemap_status['status'] === 'processing') ||
	1740	+ ($pdf_status && isset($pdf_status['status']) && $pdf_status['status'] === 'processing');
1635	1741
	1742	+ // Get single URL status, but only if no processing is active
	1743	+ $single_url_status = !$is_active_processing ? $this->mxchat_get_single_url_status() : false;
	1744	+
	1745	+ // REMOVED: Auto-clearing of completed status - now only done via dismiss button
	1746	+
1636	1747	// Return JSON response with the status data
1637	1748	wp_send_json(array(
1638	1749	'pdf_status' => $pdf_status,
1639	1750	'sitemap_status' => $sitemap_status,
	1751	+ 'single_url_status' => $single_url_status,
1640	1752	'is_processing' => $is_active_processing,
1641		- 'sitemap_queue_id' => $sitemap_queue_id,
1642		- 'pdf_queue_id' => $pdf_queue_id
	1753	+ 'current_pdf_url' => $current_pdf_url
1643	1754	));
1644	1755
1645	1756	} catch (Exception $e) {
	1757	+ // Log the error
1646	1758	//error_log('MxChat Status Update Error: ' . $e->getMessage());
1647	1759
	1760	+ // Return a friendly error response
1648	1761	wp_send_json_error(array(
1649	1762	'message' => 'Error getting status updates: ' . $e->getMessage(),
1650	1763	'status' => 'error'
1651	1764	));
		@@ -1650,1373 +1763,8 @@
1650	1763	'status' => 'error'
1651	1764	));
1652	1765	}
1653	1766	}
1654		-
1655		-/**
1656		- * Helper function to get queue status data
1657		- */
1658		-private function mxchat_get_queue_status_data($queue_id, $type = 'sitemap') {
1659		- global $wpdb;
1660		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
1661		-
1662		- // Get counts by status
1663		- $counts = $wpdb->get_results($wpdb->prepare(
1664		- "SELECT status, COUNT(*) as count
1665		- FROM $table_name
1666		- WHERE queue_id = %s
1667		- GROUP BY status",
1668		- $queue_id
1669		- ), OBJECT_K);
1670		-
1671		- $total = 0;
1672		- $completed = 0;
1673		- $failed = 0;
1674		- $processing = 0;
1675		- $pending = 0;
1676		-
1677		- foreach ($counts as $status => $data) {
1678		- $count = absint($data->count);
1679		- $total += $count;
1680		-
1681		- switch ($status) {
1682		- case 'completed':
1683		- $completed = $count;
1684		- break;
1685		- case 'failed':
1686		- $failed = $count;
1687		- break;
1688		- case 'processing':
1689		- $processing = $count;
1690		- break;
1691		- case 'pending':
1692		- $pending = $count;
1693		- break;
1694		- }
1695		- }
1696		-
1697		- if ($total === 0) {
1698		- return false;
1699		- }
1700		-
1701		- // Calculate percentage
1702		- $percentage = round((($completed + $failed) / $total) * 100);
1703		-
1704		- // Get failed items details (limit to 50)
1705		- $failed_items = array();
1706		- if ($failed > 0) {
1707		- $failed_results = $wpdb->get_results($wpdb->prepare(
1708		- "SELECT item_type, item_data, error_message, attempts, completed_at
1709		- FROM $table_name
1710		- WHERE queue_id = %s
1711		- AND status = 'failed'
1712		- AND attempts >= max_attempts
1713		- ORDER BY id DESC
1714		- LIMIT 50",
1715		- $queue_id
1716		- ));
1717		-
1718		- foreach ($failed_results as $item) {
1719		- $data = json_decode($item->item_data, true);
1720		- $url = $type === 'pdf' ? ($data['pdf_url'] ?? '') : ($data['url'] ?? '');
1721		-
1722		- $failed_items[] = array(
1723		- 'url' => $url,
1724		- 'page' => $type === 'pdf' ? ($data['page_number'] ?? 0) : 0,
1725		- 'error' => $item->error_message,
1726		- 'retries' => $item->attempts,
1727		- 'time' => strtotime($item->completed_at)
1728		- );
1729		- }
1730		- }
1731		-
1732		- // Get queue metadata
1733		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
1734		-
1735		- // Determine if queue is complete
1736		- $is_complete = ($pending === 0 && $processing === 0);
1737		-
1738		- // Get last update time
1739		- $last_update = $wpdb->get_var($wpdb->prepare(
1740		- "SELECT MAX(UNIX_TIMESTAMP(COALESCE(completed_at, started_at, created_at)))
1741		- FROM $table_name
1742		- WHERE queue_id = %s",
1743		- $queue_id
1744		- ));
1745		-
1746		- $last_update_text = $last_update ? human_time_diff($last_update, time()) . ' ' . __('ago', 'mxchat') : __('Just now', 'mxchat');
1747		-
1748		- // Format based on type
1749		- if ($type === 'pdf') {
1750		- return array(
1751		- 'total_pages' => $total,
1752		- 'processed_pages' => $completed + $failed,
1753		- 'failed_pages' => $failed,
1754		- 'percentage' => $percentage,
1755		- 'status' => $is_complete ? 'complete' : 'processing',
1756		- 'last_update' => $last_update_text,
1757		- 'failed_pages_list' => $failed_items,
1758		- 'pdf_url' => $source_url,
1759		- 'queue_id' => $queue_id
1760		- );
1761		- } else {
1762		- return array(
1763		- 'total_urls' => $total,
1764		- 'processed_urls' => $completed + $failed,
1765		- 'failed_urls' => $failed,
1766		- 'percentage' => $percentage,
1767		- 'status' => $is_complete ? 'complete' : 'processing',
1768		- 'last_update' => $last_update_text,
1769		- 'failed_urls_list' => $failed_items,
1770		- 'sitemap_url' => $source_url,
1771		- 'queue_id' => $queue_id
1772		- );
1773		- }
1774		-}
1775		-
1776		-/**
1777		- * Public method to get processing status for both sitemap and PDF queues
1778		- * Used by admin pages to display processing status
1779		- *
1780		- * @return array Array with 'sitemap_status', 'pdf_status', and 'is_processing' keys
1781		- */
1782		-public function mxchat_get_processing_statuses() {
1783		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
1784		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
1785		-
1786		- $sitemap_status = false;
1787		- $pdf_status = false;
1788		-
1789		- if ($sitemap_queue_id) {
1790		- $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
1791		- }
1792		-
1793		- if ($pdf_queue_id) {
1794		- $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
1795		- }
1796		-
1797		- $is_processing =
1798		- ($sitemap_status && $sitemap_status['status'] === 'processing') ||
1799		- ($pdf_status && $pdf_status['status'] === 'processing');
1800		-
1801		- return array(
1802		- 'sitemap_status' => $sitemap_status,
1803		- 'pdf_status' => $pdf_status,
1804		- 'is_processing' => $is_processing
1805		- );
1806		-}
1807		-
1808		-/**
1809		- * AJAX handler to get recent knowledge entries for real-time table updates
1810		- * UPDATED: Now supports both WordPress DB and Pinecone data sources
1811		- */
1812		-public function ajax_mxchat_get_recent_entries() {
1813		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
1814		-
1815		- if (!current_user_can('manage_options')) {
1816		- wp_send_json_error(array('message' => 'Unauthorized'));
1817		- return;
1818		- }
1819		-
1820		- global $wpdb;
1821		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1822		-
1823		- // Get parameters
1824		- $last_id = isset($_POST['last_id']) ? absint($_POST['last_id']) : 0;
1825		- $limit = isset($_POST['limit']) ? min(absint($_POST['limit']), 50) : 10;
1826		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
1827		-
1828		- // Check if Pinecone is enabled for this bot
1829		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
1830		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
1831		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
1832		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
1833		-
1834		- if ($use_pinecone && $has_pinecone_api) {
1835		- // PINECONE DATA SOURCE - Get grouped entry count (not raw vector count)
1836		- // Use mxchat_fetch_pinecone_records which returns total_unique_entries
1837		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, '', 1, 10, $bot_id, '');
1838		- $total_count = $records['total'] ?? 0;
1839		-
1840		- // For Pinecone, we don't return individual entries during polling
1841		- // (entries are already displayed on page load via mxchat_fetch_pinecone_records)
1842		- // We just return the updated count
1843		- wp_send_json_success(array(
1844		- 'entries' => array(),
1845		- 'total_count' => absint($total_count),
1846		- 'max_id' => $last_id,
1847		- 'data_source' => 'pinecone'
1848		- ));
1849		- return;
1850		- }
1851		-
1852		- // WORDPRESS DB DATA SOURCE
1853		- // Build query to get entries newer than last_id
1854		- $where_clauses = array('1=1');
1855		- $where_values = array();
1856		-
1857		- if ($last_id > 0) {
1858		- $where_clauses[] = 'id > %d';
1859		- $where_values[] = $last_id;
1860		- }
1861		-
1862		- // Note: WordPress DB table doesn't have bot_id column
1863		- // Multi-bot filtering is handled via Pinecone namespaces
1864		-
1865		- $where_sql = implode(' AND ', $where_clauses);
1866		-
1867		- // Get recent entries
1868		- $query = "SELECT id, article_content, source_url, timestamp
1869		- FROM $table_name
1870		- WHERE $where_sql
1871		- ORDER BY id DESC
1872		- LIMIT %d";
1873		-
1874		- $where_values[] = $limit;
1875		-
1876		- $entries = $wpdb->get_results($wpdb->prepare($query, $where_values));
1877		-
1878		- // Get total count of GROUPED entries (by source_url) - matches pagination display
1879		- // Count unique source_urls (excluding mxchat:// internal refs) + count of ungrouped rows
1880		- $total_count = $wpdb->get_var(
1881		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
1882		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
1883		- );
1884		-
1885		- // Format entries for response
1886		- $formatted_entries = array();
1887		- $preview_length = 150;
1888		- foreach ($entries as $entry) {
1889		- // Parse chunk metadata using the proper chunker method (same as initial page load)
1890		- if (class_exists('MxChat_Chunker')) {
1891		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($entry->article_content);
1892		- $display_content = $chunk_meta['text'];
1893		- $chunk_metadata = $chunk_meta['metadata'];
1894		- } else {
1895		- $display_content = $entry->article_content;
1896		- $chunk_metadata = array();
1897		- }
1898		-
1899		- $content_preview = mb_strlen($display_content) > $preview_length
1900		- ? mb_substr($display_content, 0, $preview_length) . '...'
1901		- : $display_content;
1902		-
1903		- $formatted_entries[] = array(
1904		- 'id' => $entry->id,
1905		- 'preview' => esc_html($content_preview),
1906		- 'full_content' => wp_kses_post(wpautop($display_content)),
1907		- 'content_length' => mb_strlen($display_content),
1908		- 'preview_length' => $preview_length,
1909		- 'source_url' => $entry->source_url,
1910		- 'has_link' => !empty($entry->source_url) && strpos($entry->source_url, 'mxchat://') !== 0,
1911		- 'chunk_metadata' => $chunk_metadata,
1912		- 'bot_id' => $entry->bot_id ?? 'default',
1913		- 'edit_nonce' => wp_create_nonce('mxchat_edit_entry_nonce'),
1914		- 'delete_nonce' => wp_create_nonce('mxchat_delete_prompt_nonce')
1915		- );
1916		- }
1917		-
1918		- wp_send_json_success(array(
1919		- 'entries' => $formatted_entries,
1920		- 'total_count' => absint($total_count),
1921		- 'max_id' => !empty($entries) ? $entries[0]->id : $last_id,
1922		- 'data_source' => 'wordpress'
1923		- ));
1924		-}
1925		-
1926		-/**
1927		- * Get Pinecone total count from stats API
1928		- * Helper function for ajax_mxchat_get_recent_entries
1929		- */
1930		-private function mxchat_get_pinecone_count_from_stats($pinecone_options) {
1931		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1932		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1933		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1934		-
1935		- if (empty($api_key) || empty($host)) {
1936		- return 0;
1937		- }
1938		-
1939		- try {
1940		- $stats_url = "https://{$host}/describe_index_stats";
1941		-
1942		- $response = wp_remote_post($stats_url, array(
1943		- 'headers' => array(
1944		- 'Api-Key' => $api_key,
1945		- 'Content-Type' => 'application/json'
1946		- ),
1947		- 'body' => '{}',
1948		- 'timeout' => 10
1949		- ));
1950		-
1951		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
1952		- $body = wp_remote_retrieve_body($response);
1953		- $stats_data = json_decode($body, true);
1954		-
1955		- // If namespace is specified, get count from that specific namespace
1956		- if (!empty($namespace) && isset($stats_data['namespaces'][$namespace]['vectorCount'])) {
1957		- return intval($stats_data['namespaces'][$namespace]['vectorCount']);
1958		- }
1959		-
1960		- // If no namespace specified or namespace not found in response, use total
1961		- return intval($stats_data['totalVectorCount'] ?? 0);
1962		- }
1963		-
1964		- return 0;
1965		-
1966		- } catch (Exception $e) {
1967		- return 0;
1968		- }
1969		-}
1970		-
1971		-/**
1972		- * AJAX handler to refresh Pinecone entries table via AJAX
1973		- * Returns the table HTML for updating the UI without a full page reload
1974		- */
1975		-public function ajax_mxchat_refresh_pinecone_entries() {
1976		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
1977		-
1978		- if (!current_user_can('manage_options')) {
1979		- wp_send_json_error(array('message' => 'Unauthorized'));
1980		- return;
1981		- }
1982		-
1983		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
1984		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
1985		- $per_page = 25;
1986		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
1987		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
1988		-
1989		- // Get Pinecone manager and options
1990		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
1991		- if (!$pinecone_manager) {
1992		- wp_send_json_error(array('message' => 'Pinecone manager not available'));
1993		- return;
1994		- }
1995		-
1996		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
1997		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
1998		- $pinecone_api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1999		-
2000		- if (!$use_pinecone || empty($pinecone_api_key)) {
2001		- wp_send_json_error(array('message' => 'Pinecone not configured'));
2002		- return;
2003		- }
2004		-
2005		- // Fetch records from Pinecone
2006		- $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, $search_query, $page, $per_page, $bot_id, $content_type_filter);
2007		- $prompts = $records['data'] ?? array();
2008		- $total_records = $records['total'] ?? 0;
2009		-
2010		- // Preprocess Pinecone records — set chunk_metadata and display_content
2011		- // (matches admin-knowledge-page.php preprocessing)
2012		- foreach ($prompts as $prompt) {
2013		- if (isset($prompt->chunk_index) && $prompt->chunk_index !== null) {
2014		- $prompt->chunk_metadata = array(
2015		- 'chunk_index' => intval($prompt->chunk_index),
2016		- 'total_chunks' => isset($prompt->total_chunks) ? intval($prompt->total_chunks) : null,
2017		- 'is_chunked' => isset($prompt->is_chunked) ? (bool) $prompt->is_chunked : true,
2018		- 'source_url' => $prompt->source_url ?? ''
2019		- );
2020		- $prompt->display_content = $prompt->article_content;
2021		- } else {
2022		- $prompt->chunk_metadata = array();
2023		- $prompt->display_content = $prompt->article_content ?? '';
2024		- }
2025		- }
2026		-
2027		- // Group prompts by source_url
2028		- $grouped_prompts = array();
2029		- foreach ($prompts as $prompt) {
2030		- $source_url = '';
2031		- if (!empty($prompt->chunk_metadata['source_url'])) {
2032		- $source_url = $prompt->chunk_metadata['source_url'];
2033		- } elseif (!empty($prompt->source_url)) {
2034		- $source_url = $prompt->source_url;
2035		- }
2036		-
2037		- if (!empty($source_url)) {
2038		- if (!isset($grouped_prompts[$source_url])) {
2039		- $grouped_prompts[$source_url] = array();
2040		- }
2041		- $grouped_prompts[$source_url][] = $prompt;
2042		- } else {
2043		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2044		- }
2045		- }
2046		-
2047		- // Sort each group by chunk_index
2048		- foreach ($grouped_prompts as $source_url => &$group) {
2049		- usort($group, function($a, $b) {
2050		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2051		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2052		- return $index_a - $index_b;
2053		- });
2054		- }
2055		- unset($group);
2056		-
2057		- // Build HTML for the table rows - must match admin-knowledge-page.php structure exactly
2058		- ob_start();
2059		- $display_index = 0;
2060		- $current_page = $page;
2061		- $data_source = 'pinecone';
2062		- $current_bot_id = $bot_id;
2063		- $preview_length = 150;
2064		-
2065		- if (empty($grouped_prompts)) {
2066		- echo '<tr><td colspan="4" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2067		- esc_html_e('No knowledge entries found in Pinecone.', 'mxchat');
2068		- echo '</td></tr>';
2069		- } else {
2070		- foreach ($grouped_prompts as $source_url => $group) {
2071		- $chunk_count = count($group);
2072		- $first_prompt = $group[0];
2073		- $display_index++;
2074		-
2075		- if ($chunk_count > 1) {
2076		- // Multiple chunks - show grouped row with expand button
2077		- $group_id = 'group-' . md5($source_url);
2078		- ?>
2079		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2080		- class="mxchat-chunk-group-header"
2081		- data-source="<?php echo esc_attr($data_source); ?>"
2082		- data-group-id="<?php echo esc_attr($group_id); ?>"
2083		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2084		- <td style="padding: 12px 16px; text-align: center;">
2085		- <input type="checkbox"
2086		- class="mxchat-entry-checkbox"
2087		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2088		- data-source="<?php echo esc_attr($data_source); ?>"
2089		- data-source-url="<?php echo esc_attr($source_url); ?>"
2090		- data-is-group="true"
2091		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2092		- </td>
2093		- <td style="padding: 12px 16px; font-size: 13px;">
2094		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2095		- </td>
2096		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2097		- <div class="mxchat-chunk-group-info">
2098		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2099		- <span class="dashicons dashicons-arrow-right-alt2"></span>
2100		- </button>
2101		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2102		- <span class="mxchat-chunk-preview">
2103		- <?php
2104		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : (isset($first_prompt->article_content) ? $first_prompt->article_content : '');
2105		- $content_preview = mb_substr($parent_content, 0, 100);
2106		- echo esc_html($content_preview . '...');
2107		- ?>
2108		- </span>
2109		- </div>
2110		- </td>
2111		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2112		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2113		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2114		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2115		- <?php esc_html_e('View Source', 'mxchat'); ?>
2116		- </a>
2117		- <?php else : ?>
2118		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2119		- <?php endif; ?>
2120		- </td>
2121		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2122		- <?php if ($data_source !== 'pinecone') : ?>
2123		- <button type="button"
2124		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2125		- data-source-url="<?php echo esc_attr($source_url); ?>"
2126		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2127		- data-data-source="<?php echo esc_attr($data_source); ?>"
2128		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2129		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2130		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2131		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2132		- </button>
2133		- <?php endif; ?>
2134		- <button type="button"
2135		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2136		- data-source-url="<?php echo esc_attr($source_url); ?>"
2137		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2138		- data-data-source="<?php echo esc_attr($data_source); ?>"
2139		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2140		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2141		- style="color: var(--mxch-error);"
2142		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2143		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2144		- </button>
2145		- </td>
2146		- </tr>
2147		- <?php
2148		- // Render hidden chunk rows
2149		- foreach ($group as $chunk_index => $chunk) {
2150		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
2151		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
2152		- $content = isset($chunk->display_content) ? $chunk->display_content : (isset($chunk->article_content) ? $chunk->article_content : '');
2153		- $content_preview = mb_strlen($content) > $preview_length
2154		- ? mb_substr($content, 0, $preview_length) . '...'
2155		- : $content;
2156		- ?>
2157		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
2158		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
2159		- data-source="<?php echo esc_attr($data_source); ?>"
2160		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
2161		- <td style="padding: 12px 16px; text-align: center;">
2162		- <!-- Checkbox column placeholder for chunks (managed by group) -->
2163		- </td>
2164		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
2165		- <!-- Hidden ID column for chunks -->
2166		- </td>
2167		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2168		- <div class="mxchat-accordion-wrapper">
2169		- <div class="mxchat-content-preview">
2170		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
2171		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
2172		- </span>
2173		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2174		- <?php if (mb_strlen($content) > $preview_length) : ?>
2175		- <button class="mxchat-expand-toggle" type="button">
2176		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2177		- </button>
2178		- <?php endif; ?>
2179		- </div>
2180		- <div class="mxchat-content-full" style="display: none;">
2181		- <div class="content-view">
2182		- <?php
2183		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2184		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2185		- echo wp_kses_post(wpautop($content));
2186		- echo '</div>';
2187		- } else {
2188		- echo wp_kses_post(wpautop($content));
2189		- }
2190		- ?>
2191		- </div>
2192		- </div>
2193		- </div>
2194		- </td>
2195		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2196		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
2197		- </td>
2198		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
2199		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
2200		- </td>
2201		- </tr>
2202		- <?php
2203		- }
2204		- } else {
2205		- // Single entry - display normally with accordion
2206		- $prompt = $first_prompt;
2207		- $content = isset($prompt->display_content) ? $prompt->display_content : (isset($prompt->article_content) ? $prompt->article_content : '');
2208		- $content_preview = mb_strlen($content) > $preview_length
2209		- ? mb_substr($content, 0, $preview_length) . '...'
2210		- : $content;
2211		- ?>
2212		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
2213		- data-source="<?php echo esc_attr($data_source); ?>"
2214		- style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2215		- <td style="padding: 12px 16px; text-align: center;">
2216		- <input type="checkbox"
2217		- class="mxchat-entry-checkbox"
2218		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2219		- data-source="<?php echo esc_attr($data_source); ?>"
2220		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
2221		- data-is-group="false"
2222		- data-chunk-count="1">
2223		- </td>
2224		- <td style="padding: 12px 16px; font-size: 13px;">
2225		- <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2226		- </td>
2227		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2228		- <div class="mxchat-accordion-wrapper">
2229		- <div class="mxchat-content-preview">
2230		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2231		- <?php if (mb_strlen($content) > $preview_length) : ?>
2232		- <button class="mxchat-expand-toggle" type="button">
2233		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2234		- </button>
2235		- <?php endif; ?>
2236		- </div>
2237		- <div class="mxchat-content-full" style="display: none;">
2238		- <div class="content-view">
2239		- <?php
2240		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2241		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2242		- echo wp_kses_post(wpautop($content));
2243		- echo '</div>';
2244		- } else {
2245		- echo wp_kses_post(wpautop($content));
2246		- }
2247		- ?>
2248		- </div>
2249		- </div>
2250		- </div>
2251		- </td>
2252		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2253		- <?php
2254		- $actual_source = $source_url;
2255		- if (strpos($source_url, '_ungrouped_') === 0) {
2256		- $actual_source = $prompt->source_url ?? '';
2257		- }
2258		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
2259		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2260		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2261		- <?php esc_html_e('View', 'mxchat'); ?>
2262		- </a>
2263		- <?php else : ?>
2264		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
2265		- <?php endif; ?>
2266		- </td>
2267		- <td style="padding: 12px 16px;">
2268		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-ajax" data-vector-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_pinecone_prompt_nonce'); ?>" style="color: var(--mxch-error);">
2269		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2270		- </button>
2271		- </td>
2272		- </tr>
2273		- <?php
2274		- }
2275		- }
2276		- }
2277		- $html = ob_get_clean();
2278		-
2279		- // Generate pagination HTML for Pinecone
2280		- $total_pages = ceil($total_records / $per_page);
2281		- $pagination_html = '';
2282		- if ($total_pages > 1) {
2283		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
2284		-
2285		- // Previous button
2286		- if ($page > 1) {
2287		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
2288		- }
2289		-
2290		- // Page numbers
2291		- $start_page = max(1, $page - 2);
2292		- $end_page = min($total_pages, $page + 2);
2293		-
2294		- if ($start_page > 1) {
2295		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
2296		- if ($start_page > 2) {
2297		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2298		- }
2299		- }
2300		-
2301		- for ($i = $start_page; $i <= $end_page; $i++) {
2302		- if ($i == $page) {
2303		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
2304		- } else {
2305		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
2306		- }
2307		- }
2308		-
2309		- if ($end_page < $total_pages) {
2310		- if ($end_page < $total_pages - 1) {
2311		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2312		- }
2313		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
2314		- }
2315		-
2316		- // Next button
2317		- if ($page < $total_pages) {
2318		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
2319		- }
2320		-
2321		- $pagination_html .= '</div>';
2322		- }
2323		-
2324		- wp_send_json_success(array(
2325		- 'html' => $html,
2326		- 'pagination_html' => $pagination_html,
2327		- 'total_count' => $total_records,
2328		- 'total_pages' => $total_pages,
2329		- 'page' => $page,
2330		- 'per_page' => $per_page,
2331		- 'data_source' => 'pinecone'
2332		- ));
2333		-}
2334		-
2335		-/**
2336		- * AJAX handler for pagination - handles both WordPress DB and Pinecone data sources
2337		- * Returns paginated entries without requiring a full page reload
2338		- */
2339		-public function ajax_mxchat_paginate_entries() {
2340		- check_ajax_referer('mxchat_entries_nonce', 'nonce');
2341		-
2342		- if (!current_user_can('manage_options')) {
2343		- wp_send_json_error(array('message' => 'Unauthorized'));
2344		- return;
2345		- }
2346		-
2347		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2348		- $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2349		- $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2350		- $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2351		- $per_page = 25;
2352		-
2353		- // Check if Pinecone is enabled for this bot
2354		- $pinecone_manager = $this->mxchat_get_pinecone_manager();
2355		- $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
2356		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2357		- $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
2358		-
2359		- if ($use_pinecone && $has_pinecone_api) {
2360		- // Delegate to Pinecone pagination handler (pass search params)
2361		- $_POST['page'] = $page;
2362		- $_POST['search'] = $search_query;
2363		- $_POST['content_type'] = $content_type_filter;
2364		- $this->ajax_mxchat_refresh_pinecone_entries();
2365		- return;
2366		- }
2367		-
2368		- // WordPress DB pagination - MUST match initial page load logic exactly
2369		- global $wpdb;
2370		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2371		- $offset = ($page - 1) * $per_page;
2372		-
2373		- // Build WHERE clause for search and content type filtering
2374		- $where_clauses = array();
2375		- $where_values = array();
2376		-
2377		- if ($search_query) {
2378		- $where_clauses[] = "article_content LIKE %s";
2379		- $where_values[] = '%' . $wpdb->esc_like($search_query) . '%';
2380		- }
2381		-
2382		- if ($content_type_filter) {
2383		- switch ($content_type_filter) {
2384		- case 'manual':
2385		- $where_clauses[] = "(source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')";
2386		- break;
2387		- case 'pdf':
2388		- $where_clauses[] = "source_url LIKE '%.pdf'";
2389		- break;
2390		- case 'url':
2391		- $where_clauses[] = "source_url != '' AND source_url IS NOT NULL AND source_url NOT LIKE 'mxchat://%' AND source_url NOT LIKE '%.pdf'";
2392		- break;
2393		- }
2394		- }
2395		-
2396		- $where_sql = !empty($where_clauses) ? 'WHERE ' . implode(' AND ', $where_clauses) : '';
2397		-
2398		- // Count grouped entries with filters applied
2399		- if (!empty($where_values)) {
2400		- $count_args = array_merge($where_values, $where_values);
2401		- $count_query = $wpdb->prepare(
2402		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2403		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))",
2404		- ...$count_args
2405		- );
2406		- $total_records = $wpdb->get_var($count_query);
2407		- } else if (!empty($where_sql)) {
2408		- // Content type filter only (no search), no prepared values needed
2409		- $total_records = $wpdb->get_var(
2410		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2411		- (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))"
2412		- );
2413		- } else {
2414		- // No filters
2415		- $total_records = $wpdb->get_var(
2416		- "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2417		- (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
2418		- );
2419		- }
2420		- $total_pages = ceil($total_records / $per_page);
2421		-
2422		- // Step 1: Get unique source_urls for this page (ordered by latest timestamp) with filters
2423		- if (!empty($where_values)) {
2424		- $query_args = array_merge($where_values, array($per_page, $offset));
2425		- $urls_query = $wpdb->prepare(
2426		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2427		- {$where_sql}
2428		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2429		- ...$query_args
2430		- );
2431		- } else if (!empty($where_sql)) {
2432		- $urls_query = $wpdb->prepare(
2433		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2434		- {$where_sql}
2435		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2436		- $per_page, $offset
2437		- );
2438		- } else {
2439		- $urls_query = $wpdb->prepare(
2440		- "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2441		- GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2442		- $per_page, $offset
2443		- );
2444		- }
2445		- $page_urls = $wpdb->get_results($urls_query);
2446		-
2447		- // Step 2: Build list of source_urls to fetch
2448		- $url_list = array();
2449		- $url_order_map = array();
2450		- $order_index = 0;
2451		- foreach ($page_urls as $url_row) {
2452		- $url = $url_row->source_url;
2453		- $url_list[] = $url;
2454		- $url_order_map[$url] = $order_index++;
2455		- }
2456		-
2457		- // Step 3: Fetch all rows for these source_urls (with search filter if applicable)
2458		- $prompts = array();
2459		- if (!empty($url_list)) {
2460		- $placeholders = implode(',', array_fill(0, count($url_list), '%s'));
2461		- if ($search_query) {
2462		- // Include search filter in the final fetch
2463		- $prompts_query = $wpdb->prepare(
2464		- "SELECT id, article_content, source_url, timestamp, role_restriction
2465		- FROM {$table_name}
2466		- WHERE source_url IN ($placeholders) AND article_content LIKE %s
2467		- ORDER BY timestamp DESC",
2468		- ...array_merge($url_list, ['%' . $wpdb->esc_like($search_query) . '%'])
2469		- );
2470		- } else {
2471		- $prompts_query = $wpdb->prepare(
2472		- "SELECT id, article_content, source_url, timestamp, role_restriction
2473		- FROM {$table_name}
2474		- WHERE source_url IN ($placeholders)
2475		- ORDER BY timestamp DESC",
2476		- $url_list
2477		- );
2478		- }
2479		- $prompts = $wpdb->get_results($prompts_query);
2480		- }
2481		-
2482		- // Group prompts by source_url for chunk display
2483		- $grouped_prompts = array();
2484		- foreach ($prompts as $prompt) {
2485		- $source_url = $prompt->source_url ?? '';
2486		-
2487		- // Parse chunk metadata using the proper chunker method (same as initial page load)
2488		- if (class_exists('MxChat_Chunker')) {
2489		- $chunk_meta = MxChat_Chunker::parse_stored_chunk($prompt->article_content);
2490		- $prompt->chunk_metadata = $chunk_meta['metadata'];
2491		- $prompt->display_content = $chunk_meta['text'];
2492		- } else {
2493		- $prompt->chunk_metadata = array();
2494		- $prompt->display_content = $prompt->article_content;
2495		- }
2496		-
2497		- if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0) {
2498		- if (!isset($grouped_prompts[$source_url])) {
2499		- $grouped_prompts[$source_url] = array();
2500		- }
2501		- $grouped_prompts[$source_url][] = $prompt;
2502		- } else {
2503		- // Ungrouped entries
2504		- $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2505		- }
2506		- }
2507		-
2508		- // Sort groups by the original URL order (newest first)
2509		- uksort($grouped_prompts, function($a, $b) use ($url_order_map) {
2510		- $order_a = isset($url_order_map[$a]) ? $url_order_map[$a] : PHP_INT_MAX;
2511		- $order_b = isset($url_order_map[$b]) ? $url_order_map[$b] : PHP_INT_MAX;
2512		- return $order_a - $order_b;
2513		- });
2514		-
2515		- // Sort each group internally by chunk_index
2516		- foreach ($grouped_prompts as $source_url => &$group) {
2517		- usort($group, function($a, $b) {
2518		- $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2519		- $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2520		- return $index_a - $index_b;
2521		- });
2522		- }
2523		- unset($group);
2524		-
2525		- // Build HTML for the table rows
2526		- ob_start();
2527		- $display_index = 0;
2528		- $current_page = $page;
2529		- $data_source = 'wordpress';
2530		- $current_bot_id = $bot_id;
2531		- $preview_length = 150;
2532		-
2533		- if (empty($grouped_prompts)) {
2534		- echo '<tr><td colspan="5" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2535		- esc_html_e('No knowledge entries found. Use the Import Options to add content.', 'mxchat');
2536		- echo '</td></tr>';
2537		- } else {
2538		- foreach ($grouped_prompts as $source_url => $group) {
2539		- $chunk_count = count($group);
2540		- $first_prompt = $group[0];
2541		- $display_index++;
2542		-
2543		- if ($chunk_count > 1) {
2544		- // Multiple chunks - show grouped row with expand button
2545		- $group_id = 'group-' . md5($source_url);
2546		- ?>
2547		- <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2548		- class="mxchat-chunk-group-header"
2549		- data-source="<?php echo esc_attr($data_source); ?>"
2550		- data-group-id="<?php echo esc_attr($group_id); ?>"
2551		- style="border-bottom: 1px solid var(--mxch-card-border);">
2552		- <td style="padding: 12px 16px; text-align: center;">
2553		- <input type="checkbox"
2554		- class="mxchat-entry-checkbox"
2555		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2556		- data-source="<?php echo esc_attr($data_source); ?>"
2557		- data-source-url="<?php echo esc_attr($source_url); ?>"
2558		- data-is-group="true"
2559		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2560		- </td>
2561		- <td style="padding: 12px 16px; font-size: 13px;">
2562		- <?php echo esc_html($first_prompt->id); ?>
2563		- </td>
2564		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2565		- <div class="mxchat-chunk-group-info">
2566		- <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2567		- <span class="dashicons dashicons-arrow-right-alt2"></span>
2568		- </button>
2569		- <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2570		- <span class="mxchat-chunk-preview">
2571		- <?php
2572		- $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : $first_prompt->article_content;
2573		- $content_preview = mb_substr($parent_content, 0, 100);
2574		- echo esc_html($content_preview . '...');
2575		- ?>
2576		- </span>
2577		- </div>
2578		- </td>
2579		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2580		- <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2581		- <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2582		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2583		- <?php esc_html_e('View Source', 'mxchat'); ?>
2584		- </a>
2585		- <?php else : ?>
2586		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2587		- <?php endif; ?>
2588		- </td>
2589		- <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2590		- <?php if ($data_source !== 'pinecone') : ?>
2591		- <button type="button"
2592		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2593		- data-source-url="<?php echo esc_attr($source_url); ?>"
2594		- data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2595		- data-data-source="<?php echo esc_attr($data_source); ?>"
2596		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2597		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2598		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2599		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2600		- </button>
2601		- <?php endif; ?>
2602		- <button type="button"
2603		- class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2604		- data-source-url="<?php echo esc_attr($source_url); ?>"
2605		- data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2606		- data-data-source="<?php echo esc_attr($data_source); ?>"
2607		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2608		- data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2609		- style="color: var(--mxch-error);"
2610		- title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2611		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2612		- </button>
2613		- </td>
2614		- </tr>
2615		- <?php
2616		- // Render hidden chunk rows
2617		- foreach ($group as $chunk_index => $chunk) {
2618		- $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
2619		- $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
2620		- $content = isset($chunk->display_content) ? $chunk->display_content : $chunk->article_content;
2621		- $content_preview = mb_strlen($content) > $preview_length
2622		- ? mb_substr($content, 0, $preview_length) . '...'
2623		- : $content;
2624		- ?>
2625		- <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
2626		- class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
2627		- data-source="<?php echo esc_attr($data_source); ?>"
2628		- style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
2629		- <td style="padding: 12px 16px; text-align: center;">
2630		- <!-- Checkbox column placeholder for chunks (managed by group) -->
2631		- </td>
2632		- <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
2633		- <!-- Hidden ID column for chunks -->
2634		- </td>
2635		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2636		- <div class="mxchat-accordion-wrapper">
2637		- <div class="mxchat-content-preview">
2638		- <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
2639		- <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
2640		- </span>
2641		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2642		- <?php if (mb_strlen($content) > $preview_length) : ?>
2643		- <button class="mxchat-expand-toggle" type="button">
2644		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2645		- </button>
2646		- <?php endif; ?>
2647		- </div>
2648		- <div class="mxchat-content-full" style="display: none;">
2649		- <div class="content-view">
2650		- <?php
2651		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2652		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2653		- echo wp_kses_post(wpautop($content));
2654		- echo '</div>';
2655		- } else {
2656		- echo wp_kses_post(wpautop($content));
2657		- }
2658		- ?>
2659		- </div>
2660		- </div>
2661		- </div>
2662		- </td>
2663		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2664		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
2665		- </td>
2666		- <td class="mxchat-actions-cell" style="padding: 12px 16px;">
2667		- <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
2668		- </td>
2669		- </tr>
2670		- <?php
2671		- }
2672		- } else {
2673		- // Single entry - display normally with accordion
2674		- $prompt = $first_prompt;
2675		- $content = isset($prompt->display_content) ? $prompt->display_content : $prompt->article_content;
2676		- $content_preview = mb_strlen($content) > $preview_length
2677		- ? mb_substr($content, 0, $preview_length) . '...'
2678		- : $content;
2679		- ?>
2680		- <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
2681		- data-source="<?php echo esc_attr($data_source); ?>"
2682		- style="border-bottom: 1px solid var(--mxch-card-border);">
2683		- <td style="padding: 12px 16px; text-align: center;">
2684		- <input type="checkbox"
2685		- class="mxchat-entry-checkbox"
2686		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2687		- data-source="<?php echo esc_attr($data_source); ?>"
2688		- data-source-url="<?php echo esc_attr($source_url); ?>"
2689		- data-is-group="false">
2690		- </td>
2691		- <td style="padding: 12px 16px; font-size: 13px;">
2692		- <?php echo esc_html($prompt->id); ?>
2693		- </td>
2694		- <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2695		- <div class="mxchat-accordion-wrapper">
2696		- <div class="mxchat-content-preview">
2697		- <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2698		- <?php if (mb_strlen($content) > $preview_length) : ?>
2699		- <button class="mxchat-expand-toggle" type="button">
2700		- <span class="dashicons dashicons-arrow-down-alt2"></span>
2701		- </button>
2702		- <?php endif; ?>
2703		- </div>
2704		- <div class="mxchat-content-full" style="display: none;">
2705		- <div class="content-view">
2706		- <?php
2707		- if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2708		- echo '<div dir="rtl" lang="he" class="rtl-content">';
2709		- echo wp_kses_post(wpautop($content));
2710		- echo '</div>';
2711		- } else {
2712		- echo wp_kses_post(wpautop($content));
2713		- }
2714		- ?>
2715		- </div>
2716		- </div>
2717		- </div>
2718		- </td>
2719		- <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2720		- <?php
2721		- $actual_source = $source_url;
2722		- if (strpos($source_url, '_ungrouped_') === 0) {
2723		- $actual_source = $prompt->source_url ?? '';
2724		- }
2725		- if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
2726		- <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2727		- <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2728		- <?php esc_html_e('View', 'mxchat'); ?>
2729		- </a>
2730		- <?php else : ?>
2731		- <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
2732		- <?php endif; ?>
2733		- </td>
2734		- <td style="padding: 12px 16px; white-space: nowrap;">
2735		- <button type="button"
2736		- class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2737		- data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
2738		- data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2739		- data-data-source="<?php echo esc_attr($data_source); ?>"
2740		- data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2741		- data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2742		- title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2743		- <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2744		- </button>
2745		- <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-wordpress" data-entry-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_wordpress_prompt_nonce'); ?>" style="color: var(--mxch-error);">
2746		- <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2747		- </button>
2748		- </td>
2749		- </tr>
2750		- <?php
2751		- }
2752		- }
2753		- }
2754		- $html = ob_get_clean();
2755		-
2756		- // Generate pagination HTML (include search/filter data for subsequent pages)
2757		- $pagination_html = '';
2758		- if ($total_pages > 1) {
2759		- $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
2760		-
2761		- // Previous button
2762		- if ($page > 1) {
2763		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
2764		- }
2765		-
2766		- // Page numbers
2767		- $start_page = max(1, $page - 2);
2768		- $end_page = min($total_pages, $page + 2);
2769		-
2770		- if ($start_page > 1) {
2771		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
2772		- if ($start_page > 2) {
2773		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2774		- }
2775		- }
2776		-
2777		- for ($i = $start_page; $i <= $end_page; $i++) {
2778		- if ($i == $page) {
2779		- $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
2780		- } else {
2781		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
2782		- }
2783		- }
2784		-
2785		- if ($end_page < $total_pages) {
2786		- if ($end_page < $total_pages - 1) {
2787		- $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2788		- }
2789		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
2790		- }
2791		-
2792		- // Next button
2793		- if ($page < $total_pages) {
2794		- $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
2795		- }
2796		-
2797		- $pagination_html .= '</div>';
2798		- }
2799		-
2800		- wp_send_json_success(array(
2801		- 'html' => $html,
2802		- 'pagination_html' => $pagination_html,
2803		- 'total_count' => $total_records,
2804		- 'total_pages' => $total_pages,
2805		- 'page' => $page,
2806		- 'per_page' => $per_page,
2807		- 'data_source' => 'wordpress'
2808		- ));
2809		-}
2810		-
2811		-/**
2812		- * AJAX handler to detect available sitemaps on the site
2813		- * Optimized for speed - only checks primary sitemap indexes first
2814		- */
2815		-public function ajax_mxchat_detect_sitemaps() {
2816		- check_ajax_referer('mxchat_detect_sitemaps_nonce', 'nonce');
2817		-
2818		- if (!current_user_can('manage_options')) {
2819		- wp_send_json_error(array('message' => 'Unauthorized'));
2820		- return;
2821		- }
2822		-
2823		- $site_url = get_site_url();
2824		- $sitemaps = array();
2825		- $found_index = false;
2826		-
2827		- // Only check the main sitemap index files first (much faster)
2828		- // These are the primary entry points that contain sub-sitemaps
2829		- $primary_indexes = array(
2830		- 'sitemap_index.xml' => 'Yoast SEO / Rank Math', // Yoast & Rank Math
2831		- 'wp-sitemap.xml' => 'WordPress Core', // WordPress Core
2832		- 'sitemap.xml' => 'Standard', // Generic/AIOSEO
2833		- );
2834		-
2835		- foreach ($primary_indexes as $path => $source) {
2836		- $url = trailingslashit($site_url) . $path;
2837		-
2838		- $response = wp_remote_head($url, array(
2839		- 'timeout' => 10,
2840		- 'sslverify' => false,
2841		- 'redirection' => 1,
2842		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2843		- ));
2844		-
2845		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2846		- // Found a sitemap index - parse it to get sub-sitemaps
2847		- $sub_sitemaps = $this->parse_sitemap_index($url);
2848		- if (!empty($sub_sitemaps)) {
2849		- $sitemaps[] = array(
2850		- 'url' => $url,
2851		- 'type' => 'index',
2852		- 'source' => $source,
2853		- 'sub_sitemaps' => $sub_sitemaps
2854		- );
2855		- $found_index = true;
2856		- // Found a valid index, no need to check others
2857		- break;
2858		- }
2859		- }
2860		- }
2861		-
2862		- // If no sitemap index found, check for standalone sitemaps
2863		- if (!$found_index) {
2864		- $standalone_sitemaps = array(
2865		- 'post-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
2866		- 'page-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
2867		- );
2868		-
2869		- foreach ($standalone_sitemaps as $path => $info) {
2870		- $url = trailingslashit($site_url) . $path;
2871		-
2872		- $response = wp_remote_head($url, array(
2873		- 'timeout' => 2,
2874		- 'sslverify' => false
2875		- ));
2876		-
2877		- if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2878		- $sitemaps[] = array(
2879		- 'url' => $url,
2880		- 'type' => $info['type'],
2881		- 'source' => $info['source'],
2882		- 'url_count' => 0 // Skip URL count for speed
2883		- );
2884		- }
2885		- }
2886		- }
2887		-
2888		- wp_send_json_success(array(
2889		- 'sitemaps' => $sitemaps,
2890		- 'site_url' => $site_url
2891		- ));
2892		-}
2893		-
2894		-/**
2895		- * Parse a sitemap index to get sub-sitemaps
2896		- * Optimized: doesn't fetch URL count for each sub-sitemap (too slow)
2897		- */
2898		-private function parse_sitemap_index($url) {
2899		- $sub_sitemaps = array();
2900		-
2901		- $response = wp_remote_get($url, array(
2902		- 'timeout' => 30,
2903		- 'sslverify' => false,
2904		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2905		- 'headers' => array(
2906		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
2907		- 'Accept-Language' => 'en-US,en;q=0.9',
2908		- ),
2909		- ));
2910		-
2911		- if (is_wp_error($response)) {
2912		- return $sub_sitemaps;
2913		- }
2914		-
2915		- $body = wp_remote_retrieve_body($response);
2916		- if (empty($body)) {
2917		- return $sub_sitemaps;
2918		- }
2919		-
2920		- // Suppress XML errors
2921		- libxml_use_internal_errors(true);
2922		- $xml = simplexml_load_string($body);
2923		- libxml_clear_errors();
2924		-
2925		- if ($xml === false) {
2926		- return $sub_sitemaps;
2927		- }
2928		-
2929		- // Check if it's a sitemap index (contains <sitemap> elements)
2930		- if (isset($xml->sitemap)) {
2931		- foreach ($xml->sitemap as $sitemap) {
2932		- $loc = (string) $sitemap->loc;
2933		- if (!empty($loc)) {
2934		- // Try to determine the type from the URL
2935		- $type = 'content';
2936		- if (strpos($loc, 'category') !== false || strpos($loc, 'tag') !== false || strpos($loc, 'taxonomy') !== false) {
2937		- $type = 'taxonomy';
2938		- } elseif (strpos($loc, 'author') !== false || strpos($loc, 'user') !== false) {
2939		- $type = 'author';
2940		- }
2941		-
2942		- // Skip URL count - too slow to fetch for each sitemap
2943		- $sub_sitemaps[] = array(
2944		- 'url' => $loc,
2945		- 'type' => $type,
2946		- 'url_count' => 0, // Don't fetch - takes too long
2947		- 'name' => basename(parse_url($loc, PHP_URL_PATH))
2948		- );
2949		- }
2950		- }
2951		- }
2952		-
2953		- return $sub_sitemaps;
2954		-}
2955		-
2956		-/**
2957		- * Get URL count from a sitemap
2958		- */
2959		-private function get_sitemap_url_count($url) {
2960		- $response = wp_remote_get($url, array(
2961		- 'timeout' => 30,
2962		- 'sslverify' => false,
2963		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2964		- 'headers' => array(
2965		- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
2966		- 'Accept-Language' => 'en-US,en;q=0.9',
2967		- ),
2968		- ));
2969		-
2970		- if (is_wp_error($response)) {
2971		- return 0;
2972		- }
2973		-
2974		- $body = wp_remote_retrieve_body($response);
2975		- if (empty($body)) {
2976		- return 0;
2977		- }
2978		-
2979		- // Count <url> or <loc> elements
2980		- $count = preg_match_all('/<url>/i', $body, $matches);
2981		- return $count ?: 0;
2982		-}
2983		-
2984		-/**
2985		- * Get sitemaps declared in robots.txt
2986		- */
2987		-private function get_sitemaps_from_robots($site_url) {
2988		- $sitemaps = array();
2989		- $robots_url = trailingslashit($site_url) . 'robots.txt';
2990		-
2991		- $response = wp_remote_get($robots_url, array(
2992		- 'timeout' => 15,
2993		- 'sslverify' => false,
2994		- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
2995		- ));
2996		-
2997		- if (is_wp_error($response)) {
2998		- return $sitemaps;
2999		- }
3000		-
3001		- $body = wp_remote_retrieve_body($response);
3002		- if (empty($body)) {
3003		- return $sitemaps;
3004		- }
3005		-
3006		- // Find Sitemap: declarations
3007		- if (preg_match_all('/^Sitemap:\s*(.+)$/mi', $body, $matches)) {
3008		- foreach ($matches[1] as $sitemap_url) {
3009		- $sitemap_url = trim($sitemap_url);
3010		- if (filter_var($sitemap_url, FILTER_VALIDATE_URL)) {
3011		- $sitemaps[] = $sitemap_url;
3012		- }
3013		- }
3014		- }
3015		-
3016		- return $sitemaps;
3017		-}
3018		-
3019	1767	public function mxchat_stop_processing() {
3020	1768	// Verify permissions
3021	1769	if (!current_user_can('manage_options')) {
3022	1770	wp_die(esc_html__('Unauthorized access', 'mxchat'));
		@@ -3024,52 +1772,28 @@
3024	1772
3025	1773	// Verify nonce
3026	1774	check_admin_referer('mxchat_stop_processing_action', 'mxchat_stop_processing_nonce');
3027	1775
3028		- global $wpdb;
3029		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
3030		-
3031		- // Get active queue IDs
3032		- $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
3033		- $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
3034		-
3035		- // Delete all pending items from active queues
3036		- if ($sitemap_queue_id) {
3037		- $wpdb->delete(
3038		- $table_name,
3039		- array(
3040		- 'queue_id' => $sitemap_queue_id,
3041		- 'status' => 'pending'
3042		- ),
3043		- array('%s', '%s')
3044		- );
3045		-
3046		- delete_transient('mxchat_active_queue_sitemap');
	1776	+ // Get the last sitemap URL and clear its transient
	1777	+ $sitemap_url = get_transient('mxchat_last_sitemap_url');
	1778	+ if ($sitemap_url) {
	1779	+ delete_transient('mxchat_sitemap_status_' . md5($sitemap_url));
3047	1780	delete_transient('mxchat_last_sitemap_url');
3048	1781	}
3049		-
3050		- if ($pdf_queue_id) {
3051		- // Get PDF path before deleting
3052		- $pdf_path = $this->mxchat_get_queue_meta($pdf_queue_id, 'pdf_path');
3053		-
3054		- $wpdb->delete(
3055		- $table_name,
3056		- array(
3057		- 'queue_id' => $pdf_queue_id,
3058		- 'status' => 'pending'
3059		- ),
3060		- array('%s', '%s')
3061		- );
3062		-
3063		- // Delete PDF file
3064		- if ($pdf_path && file_exists($pdf_path)) {
3065		- wp_delete_file($pdf_path);
3066		- }
3067		-
3068		- delete_transient('mxchat_active_queue_pdf');
	1782	+
	1783	+ // Get the last PDF URL and clear its transient
	1784	+ $pdf_url = get_transient('mxchat_last_pdf_url');
	1785	+ if ($pdf_url) {
	1786	+ delete_transient('mxchat_pdf_status_' . md5($pdf_url));
3069	1787	delete_transient('mxchat_last_pdf_url');
3070	1788	}
3071	1789
	1790	+ // Unschedule any pending sitemap events
	1791	+ $timestamp = wp_next_scheduled('mxchat_process_sitemap_urls');
	1792	+ if ($timestamp) {
	1793	+ wp_unschedule_event($timestamp, 'mxchat_process_sitemap_urls');
	1794	+ }
	1795	+
3072	1796	// Redirect back with a success message
3073	1797	set_transient('mxchat_admin_notice_success',
3074	1798	esc_html__('Processing has been stopped successfully.', 'mxchat'),
3075	1799	30
		@@ -3076,12 +1800,8 @@
3076	1800	);
3077	1801	wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
3078	1802	exit;
3079	1803	}
3080		-
3081		-/**
3082		- * Get content list for processing
3083		- */
3084	1804	public function ajax_mxchat_get_content_list() {
3085	1805	// Verify the nonce
3086	1806	check_ajax_referer('mxchat_content_selector_nonce', 'nonce');
3087	1807
		@@ -3089,9 +1809,9 @@
3089	1809	wp_send_json_error(__('Unauthorized access', 'mxchat'));
3090	1810	}
3091	1811
3092	1812	$page = isset($_GET['page']) ? absint($_GET['page']) : 1;
3093		- $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 100;
	1813	+ $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 20;
3094	1814	$search = isset($_GET['search']) ? sanitize_text_field($_GET['search']) : '';
3095	1815	$post_type = isset($_GET['post_type']) ? sanitize_text_field($_GET['post_type']) : 'all';
3096	1816	$post_status = isset($_GET['post_status']) ? sanitize_text_field($_GET['post_status']) : 'publish';
3097	1817	$processed_filter = isset($_GET['processed_filter']) ? sanitize_text_field($_GET['processed_filter']) : 'all';
		@@ -3104,47 +1824,20 @@
3104	1824	'orderby' => 'date',
3105	1825	'order' => 'DESC',
3106	1826	);
3107	1827
3108		- // Handle post types - IMPROVED VERSION
	1828	+ // Handle post types
3109	1829	if ($post_type !== 'all') {
3110	1830	$args['post_type'] = $post_type;
3111	1831	} else {
3112		- // Get all available post types that might contain content
3113		- $all_post_types = array();
	1832	+ // Default to post and page if we can't get post types
	1833	+ $args['post_type'] = array('post', 'page');
3114	1834
3115		- // First get all public post types
3116		- $public_types = get_post_types(array('public' => true), 'names');
3117		- $all_post_types = array_merge($all_post_types, $public_types);
3118		-
3119		- // Add common forum/community post types
3120		- $forum_types = array('topic', 'reply', 'forum', 'wpforo_topic', 'wpforo_post');
3121		- foreach ($forum_types as $forum_type) {
3122		- if (post_type_exists($forum_type)) {
3123		- $all_post_types[] = $forum_type;
3124		- }
	1835	+ // Try to get public post types
	1836	+ $public_types = $this->mxchat_get_public_post_types();
	1837	+ if (is_array($public_types) && !empty($public_types)) {
	1838	+ $args['post_type'] = array_keys($public_types);
3125	1839	}
3126		-
3127		- // Add other commonly used post types
3128		- $common_types = array('product', 'job_listing', 'event', 'portfolio');
3129		- foreach ($common_types as $common_type) {
3130		- if (post_type_exists($common_type)) {
3131		- $all_post_types[] = $common_type;
3132		- }
3133		- }
3134		-
3135		- // Remove duplicates and ensure we have at least some post types
3136		- $all_post_types = array_unique($all_post_types);
3137		-
3138		- if (empty($all_post_types)) {
3139		- // Fallback to basic post types
3140		- $all_post_types = array('post', 'page');
3141		- }
3142		-
3143		- $args['post_type'] = $all_post_types;
3144		-
3145		- // Debug logging to see what post types are being queried
3146		- //error_log('MxChat Debug: Querying post types: ' . implode(', ', $all_post_types));
3147	1840	}
3148	1841
3149	1842	if (!empty($search)) {
3150	1843	$args['s'] = $search;
		@@ -3149,58 +1842,36 @@
3149	1842	if (!empty($search)) {
3150	1843	$args['s'] = $search;
3151	1844	}
3152	1845
3153		- // Get processed data from storage
	1846	+ // ================================
	1847	+ // FIXED: Check only the ACTIVE storage method
	1848	+ // ================================
	1849	+
3154	1850	$processed_data = array();
3155	1851
	1852	+ // Check if Pinecone is enabled
3156	1853	$pinecone_options = get_option('mxchat_pinecone_addon_options', array());
3157	1854	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3158		-
	1855	+
3159	1856	if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3160		- // Get fresh data from Pinecone - no caching
	1857	+ // ONLY check Pinecone if it's enabled
3161	1858	$processed_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
3162	1859	} else {
3163		- // WordPress DB checking with better URL matching for all post types
	1860	+ // ONLY check WordPress DB if Pinecone is not enabled
3164	1861	global $wpdb;
3165	1862	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3166		- $processed_items = $wpdb->get_results("SELECT id, source_url, article_content, timestamp FROM {$table_name}");
3167		-
3168		- // Group items by source_url to count chunks
3169		- $url_chunk_counts = array();
3170		- $url_latest_timestamp = array();
3171		- $url_first_id = array();
3172		-
	1863	+ $processed_items = $wpdb->get_results("SELECT id, source_url, timestamp FROM {$table_name}");
	1864	+
3173	1865	if (!empty($processed_items)) {
3174	1866	foreach ($processed_items as $item) {
3175		- $url = $item->source_url;
3176		- if (empty($url)) continue;
3177		-
3178		- // Count chunks per URL
3179		- if (!isset($url_chunk_counts[$url])) {
3180		- $url_chunk_counts[$url] = 0;
3181		- $url_latest_timestamp[$url] = $item->timestamp;
3182		- $url_first_id[$url] = $item->id;
3183		- }
3184		- $url_chunk_counts[$url]++;
3185		-
3186		- // Track latest timestamp
3187		- if (strtotime($item->timestamp) > strtotime($url_latest_timestamp[$url])) {
3188		- $url_latest_timestamp[$url] = $item->timestamp;
3189		- }
3190		- }
3191		-
3192		- // Now build processed_data with chunk counts
3193		- foreach ($url_chunk_counts as $url => $chunk_count) {
3194		- $post_id = $this->mxchat_url_to_post_id_improved($url);
3195		-
	1867	+ $post_id = url_to_postid($item->source_url);
3196	1868	if ($post_id) {
3197	1869	$processed_data[$post_id] = array(
3198		- 'db_id' => $url_first_id[$url],
3199		- 'timestamp' => $url_latest_timestamp[$url],
3200		- 'url' => $url,
3201		- 'source' => 'wordpress',
3202		- 'chunk_count' => $chunk_count
	1870	+ 'db_id' => $item->id,
	1871	+ 'timestamp' => $item->timestamp,
	1872	+ 'url' => $item->source_url,
	1873	+ 'source' => 'wordpress'
3203	1874	);
3204	1875	}
3205	1876	}
3206	1877	}
		@@ -3205,8 +1876,10 @@
3205	1876	}
3206	1877	}
3207	1878	}
3208	1879
	1880	+ // ================================
	1881	+
3209	1882	// Get processed IDs as a simple array for in_array checks
3210	1883	$processed_ids = array_keys($processed_data);
3211	1884
3212	1885	// Handle processed/unprocessed filter
		@@ -3248,14 +1921,8 @@
3248	1921	$db_record_id = $item_data['db_id'];
3249	1922	}
3250	1923	}
3251	1924
3252		- // Get chunk count for this item
3253		- $chunk_count = 0;
3254		- if ($is_processed && isset($processed_data[$id]['chunk_count'])) {
3255		- $chunk_count = intval($processed_data[$id]['chunk_count']);
3256		- }
3257		-
3258	1925	$content_items[] = array(
3259	1926	'id' => $id,
3260	1927	'title' => get_the_title(),
3261	1928	'permalink' => get_permalink(),
		@@ -3266,10 +1933,9 @@
3266	1933	'word_count' => $word_count,
3267	1934	'already_processed' => $is_processed,
3268	1935	'processed_date' => $processed_date,
3269	1936	'db_record_id' => $db_record_id,
3270		- 'data_source' => $data_source,
3271		- 'chunk_count' => $chunk_count
	1937	+ 'data_source' => $data_source
3272	1938	);
3273	1939	}
3274	1940	wp_reset_postdata();
3275	1941	}
		@@ -3285,178 +1951,8 @@
3285	1951	wp_send_json_success($response);
3286	1952	exit;
3287	1953	}
3288	1954
3289		-
3290		-/**
3291		- * This function handles various WooCommerce URL formats and permalink structures
3292		- */
3293		-private function mxchat_url_to_post_id_improved($url) {
3294		- // First try the standard WordPress function
3295		- $post_id = url_to_postid($url);
3296		-
3297		- if ($post_id > 0) {
3298		- return $post_id;
3299		- }
3300		-
3301		- // If that fails, try more aggressive URL matching
3302		- // Remove trailing slashes and query parameters for better matching
3303		- $clean_url = rtrim($url, '/');
3304		- $clean_url = strtok($clean_url, '?'); // Remove query parameters
3305		-
3306		- // Try again with cleaned URL
3307		- $post_id = url_to_postid($clean_url);
3308		- if ($post_id > 0) {
3309		- return $post_id;
3310		- }
3311		-
3312		- // For bbPress forum topics, try extracting slug from URL
3313		- if (strpos($url, '/topic/') !== false || strpos($url, '/forums/') !== false) {
3314		- // Handle bbPress URLs: /forums/topic/topic-name/
3315		- if (preg_match('/\/forums\/topic\/([^\/\?]+)/', $url, $matches)) {
3316		- $topic_slug = $matches[1];
3317		-
3318		- // Look up topic by slug
3319		- $topic = get_page_by_path($topic_slug, OBJECT, 'topic');
3320		- if ($topic) {
3321		- return $topic->ID;
3322		- }
3323		-
3324		- // Alternative method: query by post_name
3325		- global $wpdb;
3326		- $post_id = $wpdb->get_var($wpdb->prepare(
3327		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3328		- $topic_slug
3329		- ));
3330		-
3331		- if ($post_id) {
3332		- return intval($post_id);
3333		- }
3334		- }
3335		-
3336		- // Handle simpler topic URLs: /topic/topic-name/
3337		- if (preg_match('/\/topic\/([^\/\?]+)/', $url, $matches)) {
3338		- $topic_slug = $matches[1];
3339		-
3340		- global $wpdb;
3341		- $post_id = $wpdb->get_var($wpdb->prepare(
3342		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3343		- $topic_slug
3344		- ));
3345		-
3346		- if ($post_id) {
3347		- return intval($post_id);
3348		- }
3349		- }
3350		- }
3351		-
3352		- // For WooCommerce products
3353		- if (strpos($url, '/product/') !== false || strpos($url, 'product=') !== false) {
3354		- // Extract product slug from various URL formats
3355		- $product_slug = '';
3356		-
3357		- // Handle pretty permalinks: /product/product-name/
3358		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
3359		- $product_slug = $matches[1];
3360		- }
3361		- // Handle query parameters: ?product=product-name
3362		- elseif (preg_match('/[\?&]product=([^&]+)/', $url, $matches)) {
3363		- $product_slug = $matches[1];
3364		- }
3365		-
3366		- if (!empty($product_slug)) {
3367		- // Look up product by slug
3368		- $product = get_page_by_path($product_slug, OBJECT, 'product');
3369		- if ($product) {
3370		- return $product->ID;
3371		- }
3372		-
3373		- // Alternative method: query by post_name
3374		- global $wpdb;
3375		- $post_id = $wpdb->get_var($wpdb->prepare(
3376		- "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'product' AND post_status = 'publish'",
3377		- $product_slug
3378		- ));
3379		-
3380		- if ($post_id) {
3381		- return intval($post_id);
3382		- }
3383		- }
3384		- }
3385		-
3386		- // Generic approach: try to extract slug and match against all post types
3387		- $parsed_url = wp_parse_url($clean_url);
3388		- $path = $parsed_url['path'] ?? '';
3389		-
3390		- if (!empty($path)) {
3391		- // Get the last part of the path as potential slug
3392		- $path_parts = array_filter(explode('/', trim($path, '/')));
3393		- $potential_slug = end($path_parts);
3394		-
3395		- if (!empty($potential_slug)) {
3396		- global $wpdb;
3397		-
3398		- // Try to find any post with this slug
3399		- $post_id = $wpdb->get_var($wpdb->prepare(
3400		- "SELECT ID FROM {$wpdb->posts}
3401		- WHERE post_name = %s
3402		- AND post_status IN ('publish', 'closed', 'private')
3403		- AND post_type NOT IN ('revision', 'attachment', 'nav_menu_item')
3404		- ORDER BY CASE
3405		- WHEN post_type = 'post' THEN 1
3406		- WHEN post_type = 'page' THEN 2
3407		- WHEN post_type = 'topic' THEN 3
3408		- WHEN post_type = 'product' THEN 4
3409		- ELSE 5
3410		- END
3411		- LIMIT 1",
3412		- $potential_slug
3413		- ));
3414		-
3415		- if ($post_id) {
3416		- return intval($post_id);
3417		- }
3418		- }
3419		- }
3420		-
3421		- // ADDITIONAL: Try direct database lookup by URL variations
3422		- global $wpdb;
3423		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3424		-
3425		- // Try variations of the URL (with/without trailing slash, http/https)
3426		- $url_variations = array(
3427		- $url,
3428		- rtrim($url, '/'),
3429		- $url . '/',
3430		- str_replace('http://', 'https://', $url),
3431		- str_replace('https://', 'http://', $url),
3432		- str_replace('http://', 'https://', rtrim($url, '/')),
3433		- str_replace('https://', 'http://', rtrim($url, '/'))
3434		- );
3435		-
3436		- // Remove duplicates
3437		- $url_variations = array_unique($url_variations);
3438		-
3439		- foreach ($url_variations as $variation) {
3440		- $existing_record = $wpdb->get_row($wpdb->prepare(
3441		- "SELECT id, source_url FROM $table_name WHERE source_url = %s",
3442		- $variation
3443		- ));
3444		-
3445		- if ($existing_record) {
3446		- // Try to get post ID from this stored URL
3447		- $stored_post_id = url_to_postid($existing_record->source_url);
3448		- if ($stored_post_id > 0) {
3449		- return $stored_post_id;
3450		- }
3451		- }
3452		- }
3453		-
3454		- return 0; // No match found
3455		-}
3456		-/**
3457		- * Process selected content via AJAX
3458		- */
3459	1955	public function ajax_mxchat_process_selected_content() {
3460	1956	// Basic request validation
3461	1957	if (!check_ajax_referer('mxchat_content_selector_nonce', 'nonce', false)) {
3462	1958	wp_send_json_error('Invalid nonce');
		@@ -3480,24 +1976,8 @@
3480	1976	wp_send_json_error('No content selected');
3481	1977	exit;
3482	1978	}
3483	1979
3484		- // Get bot_id from request
3485		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
3486		-
3487		- // ACF→PDF extraction is opt-in per import batch. Persist the last-used value so users
3488		- // don't re-check on every batch; the default is OFF for installs that haven't set it.
3489		- $extract_acf_pdfs = !empty($_POST['extract_acf_pdfs']) && $_POST['extract_acf_pdfs'] !== 'false';
3490		- $mxchat_options = get_option('mxchat_options', array());
3491		- if (!is_array($mxchat_options)) {
3492		- $mxchat_options = array();
3493		- }
3494		- $prior_default = !empty($mxchat_options['acf_pdf_extract_default']);
3495		- if ($prior_default !== $extract_acf_pdfs) {
3496		- $mxchat_options['acf_pdf_extract_default'] = $extract_acf_pdfs ? 1 : 0;
3497		- update_option('mxchat_options', $mxchat_options);
3498		- }
3499		-
3500	1980	// Process only ONE post at a time to avoid request size issues
3501	1981	$post_id = reset($post_ids);
3502	1982	$post = get_post($post_id);
3503	1983
		@@ -3504,185 +1984,15 @@
3504	1984	if (!$post) {
3505	1985	wp_send_json_error('Post not found');
3506	1986	exit;
3507	1987	}
3508		-
3509		- // Allow developers to modify post data before processing into knowledge base
3510		- $post = apply_filters('mxchat_before_process_post', $post, $bot_id);
3511		-
3512		- // Get content including title, short description (for WooCommerce), and main content
3513		- $content = $post->post_title . "\n\n";
3514		-
3515		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
3516		- if (!empty($post->post_excerpt)) {
3517		- // Remove shortcode tags but preserve content inside them
3518		- $clean_excerpt = $this->strip_shortcode_tags_preserve_content($post->post_excerpt);
3519		- $content .= "Short Description: " . wp_strip_all_tags($clean_excerpt) . "\n\n";
3520		- }
3521		-
3522		- // Add main content - remove shortcode tags but preserve content inside them
3523		- $clean_content = $this->strip_shortcode_tags_preserve_content($post->post_content);
3524		- $content .= wp_strip_all_tags($clean_content);
3525		-
3526		- // ADD WOOCOMMERCE PRODUCT DATA (pricing, stock, categories, custom tabs)
3527		- if (get_post_type($post_id) === 'product' && class_exists('WooCommerce')) {
3528		- $product = wc_get_product($post_id);
3529		-
3530		- if ($product) {
3531		- // Get pricing information
3532		- $regular_price = $product->get_regular_price();
3533		- $sale_price = $product->get_sale_price();
3534		- $price = $product->get_price();
3535		- $sku = $product->get_sku();
3536		-
3537		- // Get currency symbol
3538		- $currency_symbol = get_woocommerce_currency_symbol();
3539		-
3540		- // Add pricing information
3541		- $content .= "\n";
3542		- if (!empty($regular_price)) {
3543		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
3544		- } elseif (!empty($price)) {
3545		- $content .= "Price: " . $currency_symbol . $price . "\n";
3546		- }
3547		-
3548		- if (!empty($sale_price) && $sale_price !== $regular_price) {
3549		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
3550		- }
3551		-
3552		- // Handle variable products - show price range
3553		- if ($product->is_type('variable')) {
3554		- $min_price = $product->get_variation_price('min');
3555		- $max_price = $product->get_variation_price('max');
3556		- if ($min_price !== $max_price) {
3557		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
3558		- }
3559		- }
3560		-
3561		- if (!empty($sku)) {
3562		- $content .= "SKU: " . $sku . "\n";
3563		- }
3564		-
3565		- // Get product categories
3566		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
3567		- if (!empty($categories) && !is_wp_error($categories)) {
3568		- $content .= "Categories: " . implode(', ', $categories) . "\n";
3569		- }
3570		- }
3571		-
3572		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
3573		- $custom_tabs = get_post_meta($post_id, 'yikes_woo_products_tabs', true);
3574		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
3575		- foreach ($custom_tabs as $tab) {
3576		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
3577		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
3578		-
3579		- if (!empty($tab_title) && !empty($tab_content)) {
3580		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
3581		- }
3582		- }
3583		- }
3584		-
3585		- // Also check for reusable/saved tabs applied to this product
3586		- $applied_saved_tabs = get_post_meta($post_id, 'yikes_woo_reusable_products_tabs_applied', true);
3587		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
3588		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
3589		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
3590		- foreach ($applied_saved_tabs as $saved_tab_id) {
3591		- if (isset($saved_tabs[$saved_tab_id])) {
3592		- $tab = $saved_tabs[$saved_tab_id];
3593		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
3594		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
3595		-
3596		- if (!empty($tab_title) && !empty($tab_content)) {
3597		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
3598		- }
3599		- }
3600		- }
3601		- }
3602		- }
3603		- }
3604		-
3605		- // ADD ACF FIELDS SUPPORT
3606		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
3607		- $pdf_extracted_count = 0;
3608		- if (!empty($acf_fields)) {
3609		- $acf_content_parts = array();
3610		- $pdf_attachment_ids = array();
3611		-
3612		- foreach ($acf_fields as $field_name => $field_value) {
3613		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
3614		-
3615		- if (!empty($formatted_value)) {
3616		- $field_label = ucwords(str_replace('_', ' ', $field_name));
3617		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
3618		- }
3619		-
3620		- // Walk this field's value tree for any PDF attachment references and queue them for extraction.
3621		- // Only when the user opted into ACF→PDF extraction for this batch; otherwise the ACF text
3622		- // still lands in the KB but the heavier PDF parsing is skipped.
3623		- if ($extract_acf_pdfs) {
3624		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
3625		- }
3626		- }
3627		-
3628		- if (!empty($acf_content_parts)) {
3629		- $content .= "\n\n" . implode("\n", $acf_content_parts);
3630		- }
3631		-
3632		- // Extract text from each unique PDF found in ACF fields and append as a labeled section
3633		- if ($extract_acf_pdfs && !empty($pdf_attachment_ids)) {
3634		- $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
3635		- $pdf_sections = array();
3636		- foreach ($pdf_attachment_ids as $att_id) {
3637		- $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
3638		- if (!empty($pdf_text)) {
3639		- $pdf_title = get_the_title($att_id);
3640		- $pdf_url = wp_get_attachment_url($att_id);
3641		- $header = 'PDF Attachment';
3642		- if (!empty($pdf_title)) {
3643		- $header .= ': ' . $pdf_title;
3644		- }
3645		- if (!empty($pdf_url)) {
3646		- $header .= ' (' . $pdf_url . ')';
3647		- }
3648		- $pdf_sections[] = $header . "\n" . $pdf_text;
3649		- $pdf_extracted_count++;
3650		- }
3651		- }
3652		- if (!empty($pdf_sections)) {
3653		- $content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
3654		- }
3655		- }
3656		- }
3657		-
3658		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
3659		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
3660		- if (!empty($custom_meta)) {
3661		- $meta_content_parts = array();
3662		-
3663		- foreach ($custom_meta as $meta_key => $meta_value) {
3664		- // Convert meta key to readable label
3665		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
3666		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
3667		- }
3668		-
3669		- if (!empty($meta_content_parts)) {
3670		- $content .= "\n\n" . implode("\n", $meta_content_parts);
3671		- }
3672		- }
3673		-
3674		- // Debug logging for WordPress Import content
3675		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Post ID: ' . $post_id . ' Title: ' . $post->post_title);
3676		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Raw post_content length: ' . strlen($post->post_content));
3677		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Final content length: ' . strlen($content));
3678		- //error_log('[MXCHAT-WP-IMPORT-DEBUG] Content preview: ' . substr($content, 0, 300));
3679		-
3680		- // Note: Removed 10,000 char limit - chunking now handles large content properly
3681		-
3682		- // Get bot-specific API key
3683		- $bot_options = $this->get_bot_options($bot_id);
3684		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
	1988	+
	1989	+ // Get minimal content
	1990	+ $content = $post->post_title . "\n\n" . wp_strip_all_tags($post->post_content);
	1991	+ $content = substr($content, 0, 10000); // Limit content size
	1992	+
	1993	+ // Get API key with proper model detection
	1994	+ $options = get_option('mxchat_options');
3685	1995	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
3686	1996
3687	1997	if (strpos($selected_model, 'voyage') === 0) {
3688	1998	$api_key = $options['voyage_api_key'] ?? '';
		@@ -3695,9 +2005,8 @@
3695	2005	$provider_name = 'OpenAI';
3696	2006	}
3697	2007
3698	2008	if (empty($api_key)) {
3699		- MxChat_Admin::mxchat_log_debug('api_error', $provider_name . ' API key not configured for knowledge processing');
3700	2009	wp_send_json_error($provider_name . ' API key not configured');
3701	2010	exit;
3702	2011	}
3703	2012
		@@ -3703,23 +2012,26 @@
3703	2012
3704	2013	$source_url = get_permalink($post_id);
3705	2014	$vector_id = md5($source_url); // Vector ID for Pinecone
3706	2015
3707		- // Check for existing content in bot-specific storage
	2016	+ // ================================
	2017	+ // FIXED: Check for existing content in ONLY the active storage method
	2018	+ // ================================
	2019	+
3708	2020	$is_update = false;
3709	2021
3710		- // Get bot-specific Pinecone configuration
3711		- $bot_pinecone_config = $this->get_bot_pinecone_config($bot_id);
3712		- $use_pinecone = !empty($bot_pinecone_config) && ($bot_pinecone_config['use_pinecone'] ?? false);
	2022	+ // Check if Pinecone is enabled
	2023	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	2024	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3713	2025
3714		- if ($use_pinecone && !empty($bot_pinecone_config['api_key'])) {
3715		- // Check Pinecone for this bot
3716		- $pinecone_data = $this->mxchat_get_pinecone_processed_content($bot_pinecone_config);
	2026	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2027	+ // ONLY check Pinecone if it's enabled
	2028	+ $pinecone_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
3717	2029	if (isset($pinecone_data[$post_id])) {
3718	2030	$is_update = true;
3719	2031	}
3720	2032	} else {
3721		- // Check WordPress DB (same as before since it's shared)
	2033	+ // ONLY check WordPress DB if Pinecone is not enabled
3722	2034	global $wpdb;
3723	2035	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3724	2036	$existing_record = $wpdb->get_row($wpdb->prepare(
3725	2037	"SELECT id FROM $table_name WHERE source_url = %s",
		@@ -3729,54 +2041,54 @@
3729	2041	if ($existing_record) {
3730	2042	$is_update = true;
3731	2043	}
3732	2044	}
3733		-
3734		- // UPDATED 2.5.6: Determine content type based on post_type
3735		- $post_type = $post->post_type;
3736		- $content_type = 'content'; // Default fallback
3737		-
3738		- // Map WordPress post types to content types
3739		- switch ($post_type) {
3740		- case 'post':
3741		- $content_type = 'post';
3742		- break;
3743		- case 'page':
3744		- $content_type = 'page';
3745		- break;
3746		- case 'product':
3747		- $content_type = 'product';
3748		- break;
3749		- default:
3750		- // For custom post types, use the post type name
3751		- $content_type = sanitize_key($post_type);
3752		- break;
3753		- }
3754		-
3755		- // Use the centralized utility function with bot_id and content_type
	2045	+
	2046	+ // Use the centralized utility function for storage
3756	2047	$result = MxChat_Utils::submit_content_to_db(
3757		- $content,
3758		- $source_url,
	2048	+ $content,
	2049	+ $source_url,
3759	2050	$api_key,
3760		- $vector_id,
3761		- $bot_id,
3762		- $content_type
	2051	+ $vector_id
3763	2052	);
3764		-
	2053	+
3765	2054	if (is_wp_error($result)) {
3766		- MxChat_Admin::mxchat_log_debug('storage_error', 'Knowledge storage failed: ' . $result->get_error_message(), array('source_url' => $source_url));
3767	2055	wp_send_json_error('Storage failed: ' . $result->get_error_message());
3768	2056	exit;
3769	2057	}
3770		-
3771		- // Automatically apply role restriction based on tags
3772		- $this->apply_role_restriction_to_post($post_id, $source_url);
3773		-
	2058	+
	2059	+ // ================================
	2060	+ // UPDATE: Only update caches if Pinecone is enabled
	2061	+ // ================================
	2062	+
	2063	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	2064	+ // Update vector ID cache for improved fetching
	2065	+ $this->mxchat_update_pinecone_vector_cache($vector_id);
	2066	+
	2067	+ // Update local processed content cache for immediate UI feedback
	2068	+ $pinecone_cache = get_option('mxchat_pinecone_processed_cache', array());
	2069	+ $pinecone_cache[$post_id] = array(
	2070	+ 'db_id' => $vector_id,
	2071	+ 'processed_date' => 'Just now',
	2072	+ 'url' => $source_url,
	2073	+ 'source' => 'pinecone',
	2074	+ 'timestamp' => current_time('timestamp')
	2075	+ );
	2076	+ update_option('mxchat_pinecone_processed_cache', $pinecone_cache);
	2077	+
	2078	+ // Also update the general processed content cache
	2079	+ $processed_cache = get_option('mxchat_processed_content_cache', array());
	2080	+ $processed_cache[$post_id] = array(
	2081	+ 'db_id' => $vector_id,
	2082	+ 'timestamp' => current_time('timestamp'),
	2083	+ 'url' => $source_url,
	2084	+ 'source' => 'pinecone'
	2085	+ );
	2086	+ update_option('mxchat_processed_content_cache', $processed_cache);
	2087	+ }
	2088	+
3774	2089	$operation_type = $is_update ? 'update' : 'new';
3775	2090
3776		- // Count ACF fields for debugging
3777		- $acf_field_count = count($acf_fields);
3778		-
3779	2091	// Success response with minimal data
3780	2092	wp_send_json_success(array(
3781	2093	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
3782	2094	'post_id' => $post_id,
		@@ -3781,95 +2093,27 @@
3781	2093	'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
3782	2094	'post_id' => $post_id,
3783	2095	'title' => $post->post_title,
3784	2096	'operation_type' => $operation_type,
3785		- 'vector_id' => $vector_id,
3786		- 'acf_fields_found' => $acf_field_count,
3787		- 'pdf_extracted_count' => (int) $pdf_extracted_count,
3788		- 'content_preview' => substr($content, 0, 100) . '...',
3789		- 'bot_id' => $bot_id
	2097	+ 'vector_id' => $vector_id, // Include vector ID for debugging
	2098	+ 'cache_updated' => $use_pinecone // Indicate if cache was updated
3790	2099	));
3791	2100	exit;
3792	2101	}
3793	2102
3794		-private function apply_role_restriction_to_post($post_id, $source_url) {
3795		- // Get tag-role mappings
3796		- $mappings = get_option('mxchat_tag_role_mappings', array());
3797		-
3798		- if (empty($mappings)) {
3799		- return; // No mappings, leave as public
3800		- }
3801		-
3802		- // Get all tags for the post
3803		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
3804		-
3805		- if (empty($post_tags)) {
3806		- return; // No tags, leave as public
3807		- }
3808		-
3809		- // Determine the highest role restriction based on tags
3810		- $highest_role = 'public';
3811		- $role_hierarchy = array(
3812		- 'public' => 0,
3813		- 'logged_in' => 1,
3814		- 'subscriber' => 2,
3815		- 'contributor' => 3,
3816		- 'author' => 4,
3817		- 'editor' => 5,
3818		- 'administrator' => 6
3819		- );
3820		-
3821		- foreach ($post_tags as $tag_slug) {
3822		- if (isset($mappings[$tag_slug])) {
3823		- $role = $mappings[$tag_slug];
3824		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
3825		- $highest_role = $role;
3826		- }
3827		- }
3828		- }
3829		-
3830		- // If no restricted tags found, return (leave as public)
3831		- if ($highest_role === 'public') {
3832		- return;
3833		- }
3834		-
3835		- // Update the role restriction in the database
3836		- global $wpdb;
3837		-
3838		- // Check if using Pinecone
3839		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
3840		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3841		-
3842		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3843		- // Update Pinecone role restriction
3844		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
3845		- $vector_id = md5($source_url);
3846		-
3847		- $wpdb->replace(
3848		- $roles_table,
3849		- array(
3850		- 'vector_id' => $vector_id,
3851		- 'role_restriction' => $highest_role,
3852		- 'updated_at' => current_time('mysql')
3853		- ),
3854		- array('%s', '%s', '%s')
3855		- );
3856		- } else {
3857		- // Update WordPress DB
3858		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3859		-
3860		- $wpdb->update(
3861		- $table_name,
3862		- array('role_restriction' => $highest_role),
3863		- array('source_url' => $source_url),
3864		- array('%s'),
3865		- array('%s')
3866		- );
3867		- }
3868		-}
3869	2103
	2104	+
	2105	+ /**
	2106	+ * Updates cache with new vector ID if absent
	2107	+ */
	2108	+ public function mxchat_update_pinecone_vector_cache($vector_id) {
	2109	+ $cached_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2110	+ if (!in_array($vector_id, $cached_ids)) {
	2111	+ $cached_ids[] = $vector_id;
	2112	+ update_option('mxchat_pinecone_vector_ids_cache', $cached_ids);
	2113	+ }
	2114	+ }
3870	2115	public function mxchat_get_public_post_types() {
3871		- // Get all public post types
3872	2116	$post_types = get_post_types(array('public' => true), 'objects');
3873	2117	$post_type_options = array();
3874	2118
3875	2119	foreach ($post_types as $post_type) {
		@@ -3875,47 +2119,41 @@
3875	2119	foreach ($post_types as $post_type) {
3876	2120	$post_type_options[$post_type->name] = $post_type->label;
3877	2121	}
3878	2122
3879		- // Also include common forum/community post types that might not be marked as public
3880		- $additional_types = array(
3881		- 'topic' => 'Forum Topics (bbPress)',
3882		- 'reply' => 'Forum Replies (bbPress)',
3883		- 'forum' => 'Forums (bbPress)',
3884		- 'wpforo_topic' => 'wpForo Topics',
3885		- 'wpforo_post' => 'wpForo Posts'
3886		- );
3887		-
3888		- foreach ($additional_types as $type_name => $type_label) {
3889		- if (post_type_exists($type_name) && !isset($post_type_options[$type_name])) {
3890		- $post_type_options[$type_name] = $type_label;
3891		- }
3892		- }
3893		-
3894	2123	return $post_type_options;
3895	2124	}
3896		-
3897		-/**
3898		- * Retrieves processed content from Pinecone API
3899		- */
3900	2125	public function mxchat_get_pinecone_processed_content($pinecone_options) {
	2126	+ // First check local cache for immediate updates
	2127	+ $cached_data = get_option('mxchat_pinecone_processed_cache', array());
	2128	+
3901	2129	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
3902	2130	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
3903		-
	2131	+
3904	2132	if (empty($api_key) || empty($host)) {
3905		- return array();
	2133	+ // Return only cached data if API credentials are missing
	2134	+ return $cached_data;
3906	2135	}
3907		-
	2136	+
3908	2137	$pinecone_data = array();
3909		-
	2138	+
3910	2139	try {
3911		- // Always get fresh data from Pinecone
3912		- $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
3913		-
3914		- // Method 2: Final fallback - try stats endpoint (if available)
	2140	+ // Method 1: Try to get vectors using cached vector IDs first
	2141	+ $cached_vector_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
	2142	+
	2143	+ if (!empty($cached_vector_ids)) {
	2144	+ $pinecone_data = $this->mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $cached_vector_ids);
	2145	+ }
	2146	+
	2147	+ // Method 2: If no cached IDs or fetch failed, use scanning approach
3915	2148	if (empty($pinecone_data)) {
	2149	+ $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
	2150	+ }
	2151	+
	2152	+ // Method 3: Final fallback - try stats endpoint (if available)
	2153	+ if (empty($pinecone_data)) {
3916	2154	$stats_url = "https://{$host}/describe_index_stats";
3917		-
	2155	+
3918	2156	$response = wp_remote_post($stats_url, array(
3919	2157	'headers' => array(
3920	2158	'Api-Key' => $api_key,
3921	2159	'Content-Type' => 'application/json'
		@@ -3922,42 +2160,59 @@
3922	2160	),
3923	2161	'body' => json_encode(array()),
3924	2162	'timeout' => 30
3925	2163	));
3926		-
	2164	+
3927	2165	if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3928	2166	$body = wp_remote_retrieve_body($response);
3929	2167	$stats_data = json_decode($body, true);
	2168	+
	2169	+ // Log stats for debugging but don't rely on them for vector listing
	2170	+ //error_log('Pinecone index stats: ' . print_r($stats_data, true));
3930	2171	}
3931	2172	}
3932		-
	2173	+
3933	2174	} catch (Exception $e) {
3934		- // Log error but return fresh data only
	2175	+ //error_log('Pinecone processed content exception: ' . $e->getMessage());
3935	2176	}
3936		-
3937		- return $pinecone_data;
	2177	+
	2178	+ // Merge cached data with Pinecone data
	2179	+ // Cache takes priority for recent updates (within last 5 minutes)
	2180	+ $merged_data = $pinecone_data;
	2181	+
	2182	+ foreach ($cached_data as $post_id => $cache_item) {
	2183	+ $cache_timestamp = $cache_item['timestamp'] ?? 0;
	2184	+ $time_diff = current_time('timestamp') - $cache_timestamp;
	2185	+
	2186	+ // If cache item is recent (less than 5 minutes), prioritize it
	2187	+ if ($time_diff < 300) { // 5 minutes = 300 seconds
	2188	+ $merged_data[$post_id] = $cache_item;
	2189	+ } else {
	2190	+ // If not in Pinecone data and cache is old, keep cache but mark as potentially stale
	2191	+ if (!isset($merged_data[$post_id])) {
	2192	+ $merged_data[$post_id] = $cache_item;
	2193	+ }
	2194	+ }
	2195	+ }
	2196	+
	2197	+ return $merged_data;
3938	2198	}
3939	2199	public function mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $vector_ids) {
3940		- //error_log('=== DEBUG: Starting mxchat_fetch_pinecone_vectors_by_ids ===');
3941		-
3942	2200	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
3943	2201	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
3944		-
	2202	+
3945	2203	if (empty($api_key) || empty($host) || empty($vector_ids)) {
3946		- //error_log('DEBUG: Missing parameters for fetch by IDs');
3947	2204	return array();
3948	2205	}
3949		-
	2206	+
3950	2207	try {
3951	2208	$fetch_url = "https://{$host}/vectors/fetch";
3952		- //error_log('DEBUG: Fetch URL: ' . $fetch_url);
3953		- //error_log('DEBUG: Fetching ' . count($vector_ids) . ' vector IDs');
3954		-
	2209	+
3955	2210	// Pinecone fetch API allows fetching specific vectors by ID
3956	2211	$fetch_data = array(
3957	2212	'ids' => array_values($vector_ids)
3958	2213	);
3959		-
	2214	+
3960	2215	$response = wp_remote_post($fetch_url, array(
3961	2216	'headers' => array(
3962	2217	'Api-Key' => $api_key,
3963	2218	'Content-Type' => 'application/json'
		@@ -3964,45 +2219,39 @@
3964	2219	),
3965	2220	'body' => json_encode($fetch_data),
3966	2221	'timeout' => 30
3967	2222	));
3968		-
	2223	+
3969	2224	if (is_wp_error($response)) {
3970		- //error_log('DEBUG: Fetch by IDs WP error: ' . $response->get_error_message());
	2225	+ //error_log('Pinecone fetch by IDs error: ' . $response->get_error_message());
3971	2226	return array();
3972	2227	}
3973		-
	2228	+
3974	2229	$response_code = wp_remote_retrieve_response_code($response);
3975		- //error_log('DEBUG: Fetch response code: ' . $response_code);
3976		-
3977	2230	if ($response_code !== 200) {
3978		- $error_body = wp_remote_retrieve_body($response);
3979		- //error_log('DEBUG: Fetch failed with body: ' . $error_body);
	2231	+ //error_log('Pinecone fetch by IDs failed with code: ' . $response_code);
3980	2232	return array();
3981	2233	}
3982		-
	2234	+
3983	2235	$body = wp_remote_retrieve_body($response);
3984	2236	$data = json_decode($body, true);
3985	2237
3986		- //error_log('DEBUG: Fetch response structure: ' . print_r(array_keys($data), true));
3987		-
3988	2238	if (!isset($data['vectors'])) {
3989		- //error_log('DEBUG: No vectors key in response');
3990	2239	return array();
3991	2240	}
3992		-
	2241	+
3993	2242	$processed_data = array();
3994		-
	2243	+
3995	2244	foreach ($data['vectors'] as $vector_id => $vector_data) {
3996	2245	$metadata = $vector_data['metadata'] ?? array();
3997	2246	$source_url = $metadata['source_url'] ?? '';
3998		-
	2247	+
3999	2248	if (!empty($source_url)) {
4000	2249	$post_id = url_to_postid($source_url);
4001	2250	if ($post_id) {
4002	2251	$created_at = $metadata['created_at'] ?? '';
4003		- $processed_date = 'Recently';
4004		-
	2252	+ $processed_date = 'Recently'; // Default
	2253	+
4005	2254	if (!empty($created_at)) {
4006	2255	$timestamp = is_numeric($created_at) ? $created_at : strtotime($created_at);
4007	2256	if ($timestamp) {
4008	2257	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
		@@ -4007,9 +2256,9 @@
4007	2256	if ($timestamp) {
4008	2257	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
4009	2258	}
4010	2259	}
4011		-
	2260	+
4012	2261	$processed_data[$post_id] = array(
4013	2262	'db_id' => $vector_id,
4014	2263	'processed_date' => $processed_date,
4015	2264	'url' => $source_url,
		@@ -4018,90 +2267,52 @@
4018	2267	);
4019	2268	}
4020	2269	}
4021	2270	}
4022		-
4023		- //error_log('DEBUG: Processed ' . count($processed_data) . ' vectors from fetch');
	2271	+
4024	2272	return $processed_data;
4025		-
	2273	+
4026	2274	} catch (Exception $e) {
4027		- //error_log('DEBUG: Exception in fetch_pinecone_vectors_by_ids: ' . $e->getMessage());
	2275	+ //error_log('Pinecone fetch by IDs exception: ' . $e->getMessage());
4028	2276	return array();
4029	2277	}
4030	2278	}
4031		-
4032		-/**
4033		- * Get embedding dimensions based on the selected model.
4034		- */
4035		-private function mxchat_get_embedding_dimensions() {
4036		- $options = get_option('mxchat_options', array());
4037		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4038		-
4039		- $model_dimensions = array(
4040		- 'text-embedding-ada-002' => 1536,
4041		- 'text-embedding-3-small' => 1536,
4042		- 'text-embedding-3-large' => 3072,
4043		- 'voyage-2' => 1024,
4044		- 'voyage-large-2' => 1536,
4045		- 'voyage-3-large' => 2048,
4046		- 'gemini-embedding-001' => 1536,
4047		- );
4048		-
4049		- if (strpos($selected_model, 'voyage-3-large') === 0) {
4050		- $custom_dimensions = $options['voyage_output_dimension'] ?? 2048;
4051		- return intval($custom_dimensions);
4052		- }
4053		-
4054		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4055		- $custom_dimensions = $options['gemini_output_dimension'] ?? 1536;
4056		- return intval($custom_dimensions);
4057		- }
4058		-
4059		- return $model_dimensions[$selected_model] ?? 1536;
4060		-}
4061		-
4062		-/**
4063		- * Scan Pinecone for processed content
4064		- */
4065	2279	public function mxchat_scan_pinecone_for_processed_content($pinecone_options) {
4066	2280	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4067	2281	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
4068		-
	2282	+
4069	2283	if (empty($api_key) || empty($host)) {
4070	2284	return array();
4071	2285	}
4072		-
	2286	+
4073	2287	try {
4074	2288	// Use multiple random vectors to get better coverage
4075	2289	$all_matches = array();
4076	2290	$seen_ids = array();
4077		-
4078		- // Get correct dimensions for the configured embedding model
4079		- $dimensions = $this->mxchat_get_embedding_dimensions();
4080		-
	2291	+
4081	2292	// Try 3 different random vectors to get better coverage
4082	2293	for ($i = 0; $i < 3; $i++) {
4083	2294	$query_url = "https://{$host}/query";
4084		-
	2295	+
4085	2296	// Generate a random unit vector instead of zeros
4086	2297	$random_vector = array();
4087		- for ($j = 0; $j < $dimensions; $j++) {
4088		- $random_vector[] = (rand(-1000, 1000) / 1000.0);
	2298	+ for ($j = 0; $j < 1536; $j++) {
	2299	+ $random_vector[] = (rand(-1000, 1000) / 1000.0); // Random values between -1 and 1
4089	2300	}
4090		-
	2301	+
4091	2302	// Normalize the vector to unit length
4092	2303	$magnitude = sqrt(array_sum(array_map(function($x) { return $x * $x; }, $random_vector)));
4093	2304	if ($magnitude > 0) {
4094	2305	$random_vector = array_map(function($x) use ($magnitude) { return $x / $magnitude; }, $random_vector);
4095	2306	}
4096		-
	2307	+
4097	2308	$query_data = array(
4098	2309	'includeMetadata' => true,
4099	2310	'includeValues' => false,
4100		- 'topK' => 10000,
	2311	+ 'topK' => 10000, // Get many results
4101	2312	'vector' => $random_vector
4102	2313	);
4103		-
	2314	+
4104	2315	$response = wp_remote_post($query_url, array(
4105	2316	'headers' => array(
4106	2317	'Api-Key' => $api_key,
4107	2318	'Content-Type' => 'application/json'
		@@ -4108,22 +2319,21 @@
4108	2319	),
4109	2320	'body' => json_encode($query_data),
4110	2321	'timeout' => 30
4111	2322	));
4112		-
	2323	+
4113	2324	if (is_wp_error($response)) {
4114	2325	continue;
4115	2326	}
4116		-
	2327	+
4117	2328	$response_code = wp_remote_retrieve_response_code($response);
4118		-
4119	2329	if ($response_code !== 200) {
4120	2330	continue;
4121	2331	}
4122		-
	2332	+
4123	2333	$body = wp_remote_retrieve_body($response);
4124	2334	$data = json_decode($body, true);
4125		-
	2335	+
4126	2336	if (isset($data['matches'])) {
4127	2337	foreach ($data['matches'] as $match) {
4128	2338	$match_id = $match['id'] ?? '';
4129	2339	if (!empty($match_id) && !isset($seen_ids[$match_id])) {
		@@ -4132,30 +2342,24 @@
4132	2342	}
4133	2343	}
4134	2344	}
4135	2345	}
4136		-
4137		- // Convert matches to processed data format, grouping by URL to count chunks
	2346	+
	2347	+ // Convert matches to processed data format
4138	2348	$processed_data = array();
4139		- $url_chunk_counts = array();
4140		-
	2349	+ $vector_ids_for_cache = array();
	2350	+
4141	2351	foreach ($all_matches as $match) {
4142	2352	$metadata = $match['metadata'] ?? array();
4143	2353	$source_url = $metadata['source_url'] ?? '';
4144	2354	$match_id = $match['id'] ?? '';
4145		-
	2355	+
4146	2356	if (!empty($source_url) && !empty($match_id)) {
4147	2357	$post_id = url_to_postid($source_url);
4148	2358	if ($post_id) {
4149		- // Count chunks per post_id
4150		- if (!isset($url_chunk_counts[$post_id])) {
4151		- $url_chunk_counts[$post_id] = 0;
4152		- }
4153		- $url_chunk_counts[$post_id]++;
4154		-
4155	2359	$created_at = $metadata['created_at'] ?? '';
4156		- $processed_date = 'Recently';
4157		-
	2360	+ $processed_date = 'Recently'; // Default
	2361	+
4158	2362	if (!empty($created_at)) {
4159	2363	$timestamp = is_numeric($created_at) ? $created_at : strtotime($created_at);
4160	2364	if ($timestamp) {
4161	2365	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
		@@ -4160,312 +2364,242 @@
4160	2364	if ($timestamp) {
4161	2365	$processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
4162	2366	}
4163	2367	}
4164		-
4165		- // Only store if not already set, or update with newer timestamp
4166		- if (!isset($processed_data[$post_id]) ||
4167		- ($timestamp ?? 0) > ($processed_data[$post_id]['timestamp'] ?? 0)) {
4168		- $processed_data[$post_id] = array(
4169		- 'db_id' => $match_id,
4170		- 'processed_date' => $processed_date,
4171		- 'url' => $source_url,
4172		- 'source' => 'pinecone',
4173		- 'timestamp' => $timestamp ?? current_time('timestamp')
4174		- );
4175		- }
	2368	+
	2369	+ $processed_data[$post_id] = array(
	2370	+ 'db_id' => $match_id,
	2371	+ 'processed_date' => $processed_date,
	2372	+ 'url' => $source_url,
	2373	+ 'source' => 'pinecone',
	2374	+ 'timestamp' => $timestamp ?? current_time('timestamp')
	2375	+ );
	2376	+
	2377	+ $vector_ids_for_cache[] = $match_id;
4176	2378	}
4177	2379	}
4178	2380	}
4179		-
4180		- // Add chunk counts to processed data
4181		- foreach ($url_chunk_counts as $post_id => $chunk_count) {
4182		- if (isset($processed_data[$post_id])) {
4183		- $processed_data[$post_id]['chunk_count'] = $chunk_count;
4184		- }
	2381	+
	2382	+ // Update the vector IDs cache for future use
	2383	+ if (!empty($vector_ids_for_cache)) {
	2384	+ update_option('mxchat_pinecone_vector_ids_cache', $vector_ids_for_cache);
4185	2385	}
4186		-
	2386	+
4187	2387	return $processed_data;
4188		-
	2388	+
4189	2389	} catch (Exception $e) {
	2390	+ //error_log('Pinecone scan exception: ' . $e->getMessage());
4190	2391	return array();
4191	2392	}
4192		-}
4193		-/**
4194		- * Generate embeddings from input text for MXChat with bot support
4195		- */
4196		-private function mxchat_generate_embedding($text, $bot_id = 'default') {
4197		- // Enable detailed logging for debugging
4198		- //error_log('[MXCHAT-EMBED] Starting embedding generation for bot: ' . $bot_id . '. Text length: ' . strlen($text) . ' bytes');
4199		- //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
	2393	+}
4200	2394
4201		- // Get bot-specific options
4202		- $bot_options = $this->get_bot_options($bot_id);
4203		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
4204		-
4205		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4206		- //error_log('[MXCHAT-EMBED] Selected embedding model for bot ' . $bot_id . ': ' . $selected_model);
	2395	+ /**
	2396	+ * Generates embeddings from input text for MXChat
	2397	+ */
	2398	+ private function mxchat_generate_embedding($text) {
	2399	+ // Enable detailed logging for debugging
	2400	+ //error_log('[MXCHAT-EMBED] Starting embedding generation. Text length: ' . strlen($text) . ' bytes');
	2401	+ //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4207	2402
4208		- // Determine provider and endpoint
4209		- if (strpos($selected_model, 'voyage') === 0) {
4210		- $api_key = $options['voyage_api_key'] ?? '';
4211		- $endpoint = 'https://api.voyageai.com/v1/embeddings';
4212		- $provider_name = 'Voyage AI';
4213		- //error_log('[MXCHAT-EMBED] Using Voyage AI API for bot ' . $bot_id);
4214		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
4215		- $api_key = $options['gemini_api_key'] ?? '';
4216		- $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
4217		- $provider_name = 'Google Gemini';
4218		- //error_log('[MXCHAT-EMBED] Using Google Gemini API for bot ' . $bot_id);
4219		- } else {
4220		- $api_key = $options['api_key'] ?? '';
4221		- $endpoint = 'https://api.openai.com/v1/embeddings';
4222		- $provider_name = 'OpenAI';
4223		- //error_log('[MXCHAT-EMBED] Using OpenAI API for bot ' . $bot_id);
4224		- }
	2403	+ $options = get_option('mxchat_options');
	2404	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	2405	+ //error_log('[MXCHAT-EMBED] Selected embedding model: ' . $selected_model);
4225	2406
4226		- //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
	2407	+ // Determine provider and endpoint
	2408	+ if (strpos($selected_model, 'voyage') === 0) {
	2409	+ $api_key = $options['voyage_api_key'] ?? '';
	2410	+ $endpoint = 'https://api.voyageai.com/v1/embeddings';
	2411	+ $provider_name = 'Voyage AI';
	2412	+ //error_log('[MXCHAT-EMBED] Using Voyage AI API');
	2413	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	2414	+ $api_key = $options['gemini_api_key'] ?? '';
	2415	+ $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
	2416	+ $provider_name = 'Google Gemini';
	2417	+ //error_log('[MXCHAT-EMBED] Using Google Gemini API');
	2418	+ } else {
	2419	+ $api_key = $options['api_key'] ?? '';
	2420	+ $endpoint = 'https://api.openai.com/v1/embeddings';
	2421	+ $provider_name = 'OpenAI';
	2422	+ //error_log('[MXCHAT-EMBED] Using OpenAI API');
	2423	+ }
4227	2424
4228		- if (empty($api_key)) {
4229		- $error_message = sprintf('Missing %s API key for bot %s. Please configure your API key in the bot settings.', $provider_name, $bot_id);
4230		- //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
4231		- return $error_message;
4232		- }
	2425	+ //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
4233	2426
4234		- // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
4235		- $estimated_tokens = ceil(str_word_count($text) / 0.75);
4236		- //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
	2427	+ if (empty($api_key)) {
	2428	+ $error_message = sprintf('Missing %s API key. Please configure your API key in the MxChat settings.', $provider_name);
	2429	+ //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
	2430	+ return $error_message;
	2431	+ }
4237	2432
4238		- if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
4239		- //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
4240		- // Consider truncating text here
4241		- }
	2433	+ // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
	2434	+ $estimated_tokens = ceil(str_word_count($text) / 0.75);
	2435	+ //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
4242	2436
4243		- // Prepare request body based on provider
4244		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4245		- // Gemini API format
4246		- $request_body = array(
4247		- 'model' => 'models/' . $selected_model,
4248		- 'content' => array(
4249		- 'parts' => array(
4250		- array('text' => $text)
4251		- )
4252		- )
4253		- );
	2437	+ if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
	2438	+ //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
	2439	+ // Consider truncating text here
	2440	+ }
4254	2441
4255		- // Set output dimensionality to 1536 for consistency with other models
4256		- $request_body['outputDimensionality'] = 1536;
4257		- } else {
4258		- // OpenAI/Voyage API format
4259		- $request_body = array(
4260		- 'model' => $selected_model,
4261		- 'input' => $text
4262		- );
	2442	+ // Prepare request body based on provider
	2443	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2444	+ // Gemini API format
	2445	+ $request_body = array(
	2446	+ 'model' => 'models/' . $selected_model,
	2447	+ 'content' => array(
	2448	+ 'parts' => array(
	2449	+ array('text' => $text)
	2450	+ )
	2451	+ )
	2452	+ );
4263	2453
4264		- // Add output_dimension for voyage-3-large model
4265		- if ($selected_model === 'voyage-3-large') {
4266		- $request_body['output_dimension'] = 2048;
4267		- }
4268		- }
	2454	+ // Set output dimensionality to 1536 for consistency with other models
	2455	+ $request_body['outputDimensionality'] = 1536;
	2456	+ } else {
	2457	+ // OpenAI/Voyage API format
	2458	+ $request_body = array(
	2459	+ 'model' => $selected_model,
	2460	+ 'input' => $text
	2461	+ );
4269	2462
4270		- //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
	2463	+ // Add output_dimension for voyage-3-large model
	2464	+ if ($selected_model === 'voyage-3-large') {
	2465	+ $request_body['output_dimension'] = 2048;
	2466	+ }
	2467	+ }
4271	2468
4272		- // Prepare headers based on provider
4273		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4274		- // Gemini uses API key as query parameter
4275		- $endpoint .= '?key=' . $api_key;
4276		- $headers = array(
4277		- 'Content-Type' => 'application/json'
4278		- );
4279		- } else {
4280		- // OpenAI/Voyage use Bearer token
4281		- $headers = array(
4282		- 'Authorization' => 'Bearer ' . $api_key,
4283		- 'Content-Type' => 'application/json'
4284		- );
4285		- }
	2469	+ //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
4286	2470
4287		- // Make API request
4288		- //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
4289		- $response = wp_remote_post($endpoint, array(
4290		- 'body' => wp_json_encode($request_body),
4291		- 'headers' => $headers,
4292		- 'timeout' => 60 // Increased timeout for large inputs
4293		- ));
	2471	+ // Prepare headers based on provider
	2472	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2473	+ // Gemini uses API key as query parameter
	2474	+ $endpoint .= '?key=' . $api_key;
	2475	+ $headers = array(
	2476	+ 'Content-Type' => 'application/json'
	2477	+ );
	2478	+ } else {
	2479	+ // OpenAI/Voyage use Bearer token
	2480	+ $headers = array(
	2481	+ 'Authorization' => 'Bearer ' . $api_key,
	2482	+ 'Content-Type' => 'application/json'
	2483	+ );
	2484	+ }
4294	2485
4295		- // Handle wp_remote_post errors
4296		- if (is_wp_error($response)) {
4297		- $error_message = $response->get_error_message();
4298		- //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
4299		- return 'Connection error: ' . $error_message;
4300		- }
	2486	+ // Make API request
	2487	+ //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
	2488	+ $response = wp_remote_post($endpoint, array(
	2489	+ 'body' => wp_json_encode($request_body),
	2490	+ 'headers' => $headers,
	2491	+ 'timeout' => 60 // Increased timeout for large inputs
	2492	+ ));
4301	2493
4302		- // Get and check HTTP response code
4303		- $http_code = wp_remote_retrieve_response_code($response);
4304		- //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
	2494	+ // Handle wp_remote_post errors
	2495	+ if (is_wp_error($response)) {
	2496	+ $error_message = $response->get_error_message();
	2497	+ //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
	2498	+ return 'Connection error: ' . $error_message;
	2499	+ }
4305	2500
4306		- if ($http_code !== 200) {
4307		- $error_body = wp_remote_retrieve_body($response);
4308		- //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
	2501	+ // Get and check HTTP response code
	2502	+ $http_code = wp_remote_retrieve_response_code($response);
	2503	+ //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
4309	2504
4310		- // Try to parse error for more details
4311		- $error_json = json_decode($error_body, true);
4312		- if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
4313		- $error_type = $error_json['error']['type'] ?? 'unknown';
4314		- $error_message = $error_json['error']['message'] ?? 'No message';
4315		- //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
4316		- //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
	2505	+ if ($http_code !== 200) {
	2506	+ $error_body = wp_remote_retrieve_body($response);
	2507	+ //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
4317	2508
4318		- // Customize error message for common API errors
4319		- if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
4320		- $error_message = sprintf('Invalid %s API key for bot %s. Please check your API key in the bot settings.', $provider_name, $bot_id);
4321		- } elseif ($error_type === 'authentication_error') {
4322		- $error_message = sprintf('%s authentication failed for bot %s. Please verify your API key in the bot settings.', $provider_name, $bot_id);
4323		- }
	2509	+ // Try to parse error for more details
	2510	+ $error_json = json_decode($error_body, true);
	2511	+ if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
	2512	+ $error_type = $error_json['error']['type'] ?? 'unknown';
	2513	+ $error_message = $error_json['error']['message'] ?? 'No message';
	2514	+ //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
	2515	+ //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
4324	2516
4325		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4326		- return $error_message;
4327		- }
	2517	+ // Customize error message for common API errors
	2518	+ if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
	2519	+ $error_message = sprintf('Invalid %s API key. Please check your API key in the MxChat settings.', $provider_name);
	2520	+ } elseif ($error_type === 'authentication_error') {
	2521	+ $error_message = sprintf('%s authentication failed. Please verify your API key in the MxChat settings.', $provider_name);
	2522	+ }
4328	2523
4329		- $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding for bot %s", $http_code, $bot_id);
4330		- //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4331		- return $error_message;
4332		- }
	2524	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2525	+ return $error_message;
	2526	+ }
4333	2527
4334		- // Parse response body
4335		- $response_body = wp_remote_retrieve_body($response);
4336		- //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
	2528	+ $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding", $http_code);
	2529	+ //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
	2530	+ return $error_message;
	2531	+ }
4337	2532
4338		- $response_data = json_decode($response_body, true);
	2533	+ // Parse response body
	2534	+ $response_body = wp_remote_retrieve_body($response);
	2535	+ //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
4339	2536
4340		- if (json_last_error() !== JSON_ERROR_NONE) {
4341		- $error = json_last_error_msg();
4342		- //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
4343		- //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
4344		- return "Failed to parse API response: $error";
4345		- }
	2537	+ $response_data = json_decode($response_body, true);
4346	2538
4347		- // Handle different response formats based on provider
4348		- if (strpos($selected_model, 'gemini-embedding') === 0) {
4349		- // Gemini API response format
4350		- if (isset($response_data['embedding']['values'])) {
4351		- $embedding_dimensions = count($response_data['embedding']['values']);
4352		- //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
	2539	+ if (json_last_error() !== JSON_ERROR_NONE) {
	2540	+ $error = json_last_error_msg();
	2541	+ //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
	2542	+ //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
	2543	+ return "Failed to parse API response: $error";
	2544	+ }
4353	2545
4354		- // Check if embedding dimensions are as expected (should be 1536)
4355		- if ($embedding_dimensions !== 1536) {
4356		- //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
4357		- }
	2546	+ // Handle different response formats based on provider
	2547	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	2548	+ // Gemini API response format
	2549	+ if (isset($response_data['embedding']['values'])) {
	2550	+ $embedding_dimensions = count($response_data['embedding']['values']);
	2551	+ //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
4358	2552
4359		- return $response_data['embedding']['values'];
4360		- } else {
4361		- //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
4362		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2553	+ // Check if embedding dimensions are as expected (should be 1536)
	2554	+ if ($embedding_dimensions !== 1536) {
	2555	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
	2556	+ }
4363	2557
4364		- if (isset($response_data['error'])) {
4365		- $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
4366		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4367		- return $error_message;
4368		- }
	2558	+ return $response_data['embedding']['values'];
	2559	+ } else {
	2560	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
	2561	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4369	2562
4370		- $error_message = "Invalid Gemini API response format: No embedding found";
4371		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4372		- return $error_message;
4373		- }
4374		- } else {
4375		- // OpenAI/Voyage API response format
4376		- if (isset($response_data['data'][0]['embedding'])) {
4377		- $embedding_dimensions = count($response_data['data'][0]['embedding']);
4378		- //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
	2563	+ if (isset($response_data['error'])) {
	2564	+ $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
	2565	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2566	+ return $error_message;
	2567	+ }
4379	2568
4380		- // Check if embedding dimensions are as expected
4381		- if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
4382		- ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
4383		- //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
4384		- }
	2569	+ $error_message = "Invalid Gemini API response format: No embedding found";
	2570	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2571	+ return $error_message;
	2572	+ }
	2573	+ } else {
	2574	+ // OpenAI/Voyage API response format
	2575	+ if (isset($response_data['data'][0]['embedding'])) {
	2576	+ $embedding_dimensions = count($response_data['data'][0]['embedding']);
	2577	+ //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
4385	2578
4386		- return $response_data['data'][0]['embedding'];
4387		- } else {
4388		- //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
4389		- //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
	2579	+ // Check if embedding dimensions are as expected
	2580	+ if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
	2581	+ ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
	2582	+ //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
	2583	+ }
4390	2584
4391		- if (isset($response_data['error'])) {
4392		- $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
4393		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4394		- return $error_message;
4395		- }
	2585	+ return $response_data['data'][0]['embedding'];
	2586	+ } else {
	2587	+ //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
	2588	+ //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4396	2589
4397		- $error_message = "Invalid API response format: No embedding found";
4398		- //error_log('[MXCHAT-EMBED] ' . $error_message);
4399		- return $error_message;
4400		- }
4401		- }
4402		-}
	2590	+ if (isset($response_data['error'])) {
	2591	+ $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
	2592	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2593	+ return $error_message;
	2594	+ }
4403	2595
4404		-/**
4405		- * Get bot-specific options for multi-bot functionality
4406		- * Falls back to default options if bot_id is 'default' or multi-bot add-on is not active
4407		- */
4408		-private function get_bot_options($bot_id = 'default') {
4409		- //error_log("MXCHAT DEBUG: get_bot_options called for bot: " . $bot_id);
4410		-
4411		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4412		- //error_log("MXCHAT DEBUG: Using default options (no multi-bot or bot is 'default')");
4413		- return array();
4414		- }
4415		-
4416		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
4417		-
4418		- if (!empty($bot_options)) {
4419		- //error_log("MXCHAT DEBUG: Got bot-specific options from filter");
4420		- if (isset($bot_options['similarity_threshold'])) {
4421		- //error_log(" - similarity_threshold: " . $bot_options['similarity_threshold']);
4422		- }
4423		- }
4424		-
4425		- return is_array($bot_options) ? $bot_options : array();
4426		-}
4427		-
4428		-/**
4429		- * Get bot-specific Pinecone configuration
4430		- * Used in the knowledge retrieval functions
4431		- */
4432		-// Also add debugging to your get_bot_pinecone_config function
4433		-private function get_bot_pinecone_config($bot_id = 'default') {
4434		- //error_log("MXCHAT DEBUG: get_bot_pinecone_config called for bot: " . $bot_id);
4435		-
4436		- // If default bot or multi-bot add-on not active, use default Pinecone config
4437		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4438		- //error_log("MXCHAT DEBUG: Using default Pinecone config (no multi-bot or bot is 'default')");
4439		- $addon_options = get_option('mxchat_pinecone_addon_options', array());
4440		- $config = array(
4441		- 'use_pinecone' => (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1'),
4442		- 'api_key' => $addon_options['mxchat_pinecone_api_key'] ?? '',
4443		- 'host' => $addon_options['mxchat_pinecone_host'] ?? '',
4444		- 'namespace' => $addon_options['mxchat_pinecone_namespace'] ?? ''
4445		- );
4446		- //error_log("MXCHAT DEBUG: Default config - use_pinecone: " . ($config['use_pinecone'] ? 'true' : 'false'));
4447		- return $config;
4448		- }
4449		-
4450		- //error_log("MXCHAT DEBUG: Calling filter 'mxchat_get_bot_pinecone_config' for bot: " . $bot_id);
4451		-
4452		- // Hook for multi-bot add-on to provide bot-specific Pinecone config
4453		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
4454		-
4455		- if (!empty($bot_pinecone_config)) {
4456		- //error_log("MXCHAT DEBUG: Got bot-specific config from filter");
4457		- //error_log(" - use_pinecone: " . (isset($bot_pinecone_config['use_pinecone']) ? ($bot_pinecone_config['use_pinecone'] ? 'true' : 'false') : 'not set'));
4458		- //error_log(" - host: " . ($bot_pinecone_config['host'] ?? 'not set'));
4459		- //error_log(" - namespace: " . ($bot_pinecone_config['namespace'] ?? 'not set'));
4460		- } else {
4461		- //error_log("MXCHAT DEBUG: Filter returned empty config!");
4462		- }
4463		-
4464		- return is_array($bot_pinecone_config) ? $bot_pinecone_config : array();
4465		-}
4466		-
4467		-
	2596	+ $error_message = "Invalid API response format: No embedding found";
	2597	+ //error_log('[MXCHAT-EMBED] ' . $error_message);
	2598	+ return $error_message;
	2599	+ }
	2600	+ }
	2601	+ }
4468	2602	public function mxchat_ajax_dismiss_completed_status() {
4469	2603	try {
4470	2604	// Verify the request
4471	2605	check_ajax_referer('mxchat_status_nonce', 'nonce');
		@@ -4584,10 +2718,10 @@
4584	2718
4585	2719	$html .= '<p><strong>' . esc_html__('Status:', 'mxchat') . '</strong> ' . esc_html(ucfirst($status['status'])) . '</p>';
4586	2720	$html .= '<p><strong>' . esc_html__('Last update:', 'mxchat') . '</strong> ' . esc_html($status['last_update']) . '</p>';
4587	2721
4588		- // Add completion summary if available AND it's an array
4589		- if (isset($status['completion_summary']) && is_array($status['completion_summary']) && !empty($status['completion_summary'])) {
	2722	+ // Add completion summary if available
	2723	+ if (isset($status['completion_summary'])) {
4590	2724	$summary = $status['completion_summary'];
4591	2725	$html .= '<div class="mxchat-completion-summary">';
4592	2726	$html .= '<h5>' . esc_html__('Processing Summary', 'mxchat') . '</h5>';
4593	2727	$html .= '<p><strong>' . esc_html__('Total Pages:', 'mxchat') . '</strong> ' . esc_html($summary['total_pages']) . '</p>';
		@@ -4596,10 +2730,10 @@
4596	2730	$html .= '<p><strong>' . esc_html__('Completed:', 'mxchat') . '</strong> ' . esc_html($summary['completion_time']) . '</p>';
4597	2731	$html .= '</div>';
4598	2732	}
4599	2733
4600		- // Add failed pages list if any AND it's an array
4601		- if (isset($status['failed_pages_list']) && is_array($status['failed_pages_list']) && !empty($status['failed_pages_list'])) {
	2734	+ // Add failed pages list if any
	2735	+ if (!empty($status['failed_pages_list'])) {
4602	2736	$html .= $this->mxchat_render_failed_pages_list($status['failed_pages_list']);
4603	2737	}
4604	2738
4605	2739	// Add error message if any
		@@ -4613,8 +2747,9 @@
4613	2747	$html .= '</div>'; // End card
4614	2748
4615	2749	return $html;
4616	2750	}
	2751	+
4617	2752	/**
4618	2753	* Render sitemap status card HTML
4619	2754	*/
4620	2755	private function mxchat_render_sitemap_status_card($status, $sitemap_url) {
		@@ -4670,10 +2805,10 @@
4670	2805
4671	2806	$html .= '<p><strong>' . esc_html__('Status:', 'mxchat') . '</strong> ' . esc_html(ucfirst($status['status'])) . '</p>';
4672	2807	$html .= '<p><strong>' . esc_html__('Last update:', 'mxchat') . '</strong> ' . esc_html($status['last_update']) . '</p>';
4673	2808
4674		- // Add completion summary if available AND it's an array
4675		- if (isset($status['completion_summary']) && is_array($status['completion_summary']) && !empty($status['completion_summary'])) {
	2809	+ // Add completion summary if available
	2810	+ if (isset($status['completion_summary'])) {
4676	2811	$summary = $status['completion_summary'];
4677	2812	$html .= '<div class="mxchat-completion-summary">';
4678	2813	$html .= '<h5>' . esc_html__('Processing Summary', 'mxchat') . '</h5>';
4679	2814	$html .= '<p><strong>' . esc_html__('Total URLs:', 'mxchat') . '</strong> ' . esc_html($summary['total_urls']) . '</p>';
		@@ -4702,16 +2837,13 @@
4702	2837	$html .= '</div>'; // End card
4703	2838
4704	2839	return $html;
4705	2840	}
4706		-
4707		-
4708	2841	/**
4709	2842	* Render failed pages list
4710	2843	*/
4711	2844	private function mxchat_render_failed_pages_list($failed_pages_list) {
4712		- // Validate that $failed_pages_list is an array and not empty
4713		- if (!is_array($failed_pages_list) || empty($failed_pages_list)) {
	2845	+ if (empty($failed_pages_list)) {
4714	2846	return '';
4715	2847	}
4716	2848
4717	2849	$html = '<div class="mxchat-error-notice">';
		@@ -4736,13 +2868,8 @@
4736	2868	return ($b['time'] ?? 0) - ($a['time'] ?? 0);
4737	2869	});
4738	2870
4739	2871	foreach ($sorted_failed_pages as $item) {
4740		- // Ensure $item is an array before accessing its elements
4741		- if (!is_array($item)) {
4742		- continue;
4743		- }
4744		-
4745	2872	$time_ago = isset($item['time']) ? human_time_diff($item['time'], current_time('timestamp')) . ' ' . esc_html__('ago', 'mxchat') : 'Unknown';
4746	2873	$html .= '<tr>';
4747	2874	$html .= '<td>' . esc_html__('Page', 'mxchat') . ' ' . esc_html($item['page'] ?? 'Unknown') . '</td>';
4748	2875	$html .= '<td style="word-break: break-word;">' . esc_html($item['error'] ?? 'Unknown error') . '</td>';
		@@ -4760,10 +2887,9 @@
4760	2887	/**
4761	2888	* Render failed URLs list
4762	2889	*/
4763	2890	private function mxchat_render_failed_urls_list($failed_urls_list) {
4764		- // Validate that $failed_urls_list is an array and not empty
4765		- if (!is_array($failed_urls_list) || empty($failed_urls_list)) {
	2891	+ if (empty($failed_urls_list)) {
4766	2892	return '';
4767	2893	}
4768	2894
4769	2895	$html = '<div class="mxchat-failed-urls-container">';
		@@ -4790,13 +2916,8 @@
4790	2916	// Show up to 50 failed URLs
4791	2917	$display_urls = array_slice($sorted_failed_urls, 0, 50);
4792	2918
4793	2919	foreach ($display_urls as $item) {
4794		- // Ensure $item is an array before accessing its elements
4795		- if (!is_array($item)) {
4796		- continue;
4797		- }
4798		-
4799	2920	$url = $item['url'] ?? '';
4800	2921	$time_ago = isset($item['time']) ? human_time_diff($item['time'], current_time('timestamp')) . ' ' . esc_html__('ago', 'mxchat') : 'Unknown';
4801	2922
4802	2923	// Truncate URL for display
		@@ -4828,564 +2949,18 @@
4828	2949
4829	2950	return $html;
4830	2951	}
4831	2952
4832		-/**
4833		- * Get all ACF fields for a specific post, excluding any fields the user has disabled
4834		- */
4835		-public function mxchat_get_acf_fields_for_post($post_id) {
4836		- if (!function_exists('get_fields')) {
4837		- return array();
4838		- }
4839		-
4840		- $fields = get_fields($post_id);
4841		- if (!$fields || !is_array($fields)) {
4842		- return array();
4843		- }
4844		-
4845		- // Get excluded fields from settings
4846		- $excluded_fields = get_option('mxchat_acf_excluded_fields', array());
4847		- if (!empty($excluded_fields) && is_array($excluded_fields)) {
4848		- foreach ($excluded_fields as $excluded_field) {
4849		- if (isset($fields[$excluded_field])) {
4850		- unset($fields[$excluded_field]);
4851		- }
4852		- }
4853		- }
4854		-
4855		- return $fields;
4856		-}
4857		-
4858		-/**
4859		- * Get all registered ACF field groups and their fields for the settings UI
4860		- */
4861		-public function mxchat_get_all_acf_fields() {
4862		- if (!function_exists('acf_get_field_groups') || !function_exists('acf_get_fields')) {
4863		- return array();
4864		- }
4865		-
4866		- $all_fields = array();
4867		- $field_groups = acf_get_field_groups();
4868		-
4869		- if (!empty($field_groups)) {
4870		- foreach ($field_groups as $group) {
4871		- $group_fields = acf_get_fields($group['key']);
4872		- if (!empty($group_fields)) {
4873		- $all_fields[$group['title']] = array();
4874		- foreach ($group_fields as $field) {
4875		- $all_fields[$group['title']][] = array(
4876		- 'name' => $field['name'],
4877		- 'label' => $field['label'],
4878		- 'type' => $field['type']
4879		- );
4880		- }
4881		- }
4882		- }
4883		- }
4884		-
4885		- return $all_fields;
4886		-}
4887		-
4888		-/**
4889		- * Get whitelisted custom post meta for a given post
4890		- * This allows non-ACF meta fields (like OptionTree, theme meta boxes) to be included in embeddings
4891		- */
4892		-public function mxchat_get_whitelisted_post_meta($post_id) {
4893		- $whitelist = get_option('mxchat_custom_meta_whitelist', '');
4894		-
4895		- if (empty($whitelist)) {
4896		- return array();
4897		- }
4898		-
4899		- // Parse the whitelist - one meta key per line
4900		- $meta_keys = array_filter(array_map('trim', explode("\n", $whitelist)));
4901		-
4902		- if (empty($meta_keys)) {
4903		- return array();
4904		- }
4905		-
4906		- $result = array();
4907		-
4908		- foreach ($meta_keys as $key) {
4909		- // Skip empty keys
4910		- if (empty($key)) {
4911		- continue;
4912		- }
4913		-
4914		- $value = get_post_meta($post_id, $key, true);
4915		-
4916		- // Only include non-empty string values
4917		- if (!empty($value) && is_string($value)) {
4918		- $result[$key] = $value;
4919		- } elseif (!empty($value) && is_array($value)) {
4920		- // Handle array values by joining them
4921		- $flat_value = $this->mxchat_flatten_meta_array($value);
4922		- if (!empty($flat_value)) {
4923		- $result[$key] = $flat_value;
4924		- }
4925		- }
4926		- }
4927		-
4928		- return $result;
4929		-}
4930		-
4931		-/**
4932		- * Flatten array meta values into a readable string
4933		- */
4934		-private function mxchat_flatten_meta_array($array, $depth = 0) {
4935		- if ($depth > 3) {
4936		- return ''; // Prevent infinite recursion
4937		- }
4938		-
4939		- $parts = array();
4940		-
4941		- foreach ($array as $key => $value) {
4942		- if (is_string($value) && !empty($value)) {
4943		- $parts[] = $value;
4944		- } elseif (is_array($value)) {
4945		- $nested = $this->mxchat_flatten_meta_array($value, $depth + 1);
4946		- if (!empty($nested)) {
4947		- $parts[] = $nested;
4948		- }
4949		- }
4950		- }
4951		-
4952		- return implode(', ', $parts);
4953		-}
4954		-
4955		-/**
4956		- * Format ACF field values for content extraction
4957		- */
4958		-public function mxchat_format_acf_field_value($value, $field_name = '', $post_id = 0) {
4959		- if (empty($value)) {
4960		- return '';
4961		- }
4962		-
4963		- // Handle WP_Post objects first (THIS IS THE KEY FIX)
4964		- if ($value instanceof WP_Post) {
4965		- return $value->post_title ?: '';
4966		- }
4967		-
4968		- // Handle other WP objects
4969		- if (is_object($value)) {
4970		- if (isset($value->post_title)) {
4971		- return $value->post_title;
4972		- } elseif (isset($value->display_name)) {
4973		- return $value->display_name;
4974		- } elseif (isset($value->name)) {
4975		- return $value->name;
4976		- } elseif (method_exists($value, '__toString')) {
4977		- try {
4978		- return (string) $value;
4979		- } catch (Exception $e) {
4980		- return '';
4981		- }
4982		- }
4983		- // For any other objects, return empty string
4984		- return '';
4985		- }
4986		-
4987		- // Handle different ACF field types
4988		- if (is_array($value)) {
4989		- // Check if it's an image/file field
4990		- if (isset($value['url'])) {
4991		- // Image field - return alt text, title, or caption
4992		- if (!empty($value['alt'])) {
4993		- return $value['alt'];
4994		- } elseif (!empty($value['title'])) {
4995		- return $value['title'];
4996		- } elseif (!empty($value['caption'])) {
4997		- return $value['caption'];
4998		- } else {
4999		- return ''; // Don't include just the URL
5000		- }
5001		- }
5002		-
5003		- // Check if it's a post object or relationship field
5004		- if (isset($value['post_title'])) {
5005		- return $value['post_title'];
5006		- }
5007		-
5008		- // Check if it's a user field
5009		- if (isset($value['display_name'])) {
5010		- return $value['display_name'];
5011		- }
5012		-
5013		- // Check if it's a taxonomy term
5014		- if (isset($value['name']) && isset($value['taxonomy'])) {
5015		- return $value['name'];
5016		- }
5017		-
5018		- // Check if it's a select field with label
5019		- if (isset($value['label'])) {
5020		- return $value['label'];
5021		- }
5022		-
5023		- // Check for repeater field or flexible content
5024		- if (is_numeric(key($value))) {
5025		- $sub_values = array();
5026		- foreach ($value as $sub_item) {
5027		- if (is_array($sub_item)) {
5028		- // For repeater/flexible content, extract text values
5029		- $sub_text = $this->mxchat_extract_text_from_acf_array($sub_item);
5030		- if (!empty($sub_text)) {
5031		- $sub_values[] = $sub_text;
5032		- }
5033		- } elseif ($sub_item instanceof WP_Post) {
5034		- // Handle WP_Post objects in arrays
5035		- $sub_values[] = $sub_item->post_title ?: '';
5036		- } else {
5037		- $sub_values[] = (string) $sub_item;
5038		- }
5039		- }
5040		- return implode(', ', array_filter($sub_values));
5041		- }
5042		-
5043		- // For other arrays, try to extract meaningful text
5044		- $text_values = array();
5045		- foreach ($value as $key => $val) {
5046		- if (is_string($val) && !empty(trim($val))) {
5047		- $text_values[] = trim($val);
5048		- } elseif ($val instanceof WP_Post) {
5049		- // Handle WP_Post objects in associative arrays
5050		- $text_values[] = $val->post_title ?: '';
5051		- } elseif (is_array($val) && isset($val['post_title'])) {
5052		- $text_values[] = $val['post_title'];
5053		- } elseif (is_array($val) && isset($val['name'])) {
5054		- $text_values[] = $val['name'];
5055		- }
5056		- }
5057		-
5058		- return implode(', ', array_filter($text_values));
5059		- }
5060		-
5061		- // Handle boolean values
5062		- if (is_bool($value)) {
5063		- return $value ? 'Yes' : 'No';
5064		- }
5065		-
5066		- // Handle numeric values
5067		- if (is_numeric($value)) {
5068		- return (string) $value;
5069		- }
5070		-
5071		- // Handle string values
5072		- if (is_string($value)) {
5073		- return trim($value);
5074		- }
5075		-
5076		- // For anything else that we can't handle, return empty string
5077		- // This prevents the "Object could not be converted to string" error
5078		- return '';
5079		-}
5080		-
5081		-/**
5082		- * Extract text from complex ACF array structures
5083		- */
5084		-private function mxchat_extract_text_from_acf_array($array) {
5085		- if (!is_array($array)) {
5086		- return '';
5087		- }
5088		-
5089		- $text_parts = array();
5090		-
5091		- foreach ($array as $key => $value) {
5092		- if (is_string($value) && !empty(trim($value))) {
5093		- // Skip keys that are likely to be IDs or technical values
5094		- if (!is_numeric($value) || strlen($value) > 10) {
5095		- $text_parts[] = trim($value);
5096		- }
5097		- } elseif ($value instanceof WP_Post) {
5098		- // Handle WP_Post objects
5099		- $text_parts[] = $value->post_title ?: '';
5100		- } elseif (is_array($value)) {
5101		- if (isset($value['post_title'])) {
5102		- $text_parts[] = $value['post_title'];
5103		- } elseif (isset($value['name'])) {
5104		- $text_parts[] = $value['name'];
5105		- } elseif (isset($value['label'])) {
5106		- $text_parts[] = $value['label'];
5107		- }
5108		- } elseif (is_object($value)) {
5109		- // Handle other objects safely
5110		- if (isset($value->post_title)) {
5111		- $text_parts[] = $value->post_title;
5112		- } elseif (isset($value->name)) {
5113		- $text_parts[] = $value->name;
5114		- } elseif (isset($value->display_name)) {
5115		- $text_parts[] = $value->display_name;
5116		- }
5117		- }
5118		- }
5119		-
5120		- return implode(', ', array_filter($text_parts));
5121		-}
5122		-
5123		-/**
5124		- * Walk an ACF field value tree and collect attachment IDs for any value that
5125		- * resolves to a PDF in the WordPress media library. Handles the three shapes
5126		- * ACF returns for File/Image/URL fields (array with ID+url, integer attachment ID,
5127		- * plain URL string), and recurses through repeater/group/flexible content.
5128		- *
5129		- * @param mixed $value The ACF field value (any depth)
5130		- * @param array $out Accumulator (passed by reference) for attachment IDs
5131		- * @param int $depth Recursion guard
5132		- */
5133		-private function mxchat_collect_pdf_attachment_ids_from_acf_value($value, &$out, $depth = 0) {
5134		- if ($depth > 6) {
5135		- return; // prevent runaway recursion on circular/very-deep structures
5136		- }
5137		-
5138		- if (empty($value)) {
5139		- return;
5140		- }
5141		-
5142		- // Array shapes: ACF File/Image return value=array; repeaters/groups are arrays of arrays
5143		- if (is_array($value)) {
5144		- // Direct File/Image-style array (has 'url' and usually 'ID' + 'mime_type')
5145		- $looks_like_attachment = isset($value['url']) || isset($value['ID']) || isset($value['id']);
5146		- if ($looks_like_attachment) {
5147		- $att_id = 0;
5148		- if (!empty($value['ID']) && is_numeric($value['ID'])) {
5149		- $att_id = (int) $value['ID'];
5150		- } elseif (!empty($value['id']) && is_numeric($value['id'])) {
5151		- $att_id = (int) $value['id'];
5152		- } elseif (!empty($value['url']) && is_string($value['url'])) {
5153		- $att_id = (int) attachment_url_to_postid($value['url']);
5154		- }
5155		-
5156		- $is_pdf = false;
5157		- if (!empty($value['mime_type']) && $value['mime_type'] === 'application/pdf') {
5158		- $is_pdf = true;
5159		- } elseif (!empty($value['subtype']) && strtolower((string) $value['subtype']) === 'pdf') {
5160		- $is_pdf = true;
5161		- } elseif (!empty($value['url']) && is_string($value['url']) && $this->mxchat_url_looks_like_pdf($value['url'])) {
5162		- $is_pdf = true;
5163		- } elseif ($att_id && get_post_mime_type($att_id) === 'application/pdf') {
5164		- $is_pdf = true;
5165		- }
5166		-
5167		- if ($is_pdf && $att_id && get_post_mime_type($att_id) === 'application/pdf') {
5168		- $out[] = $att_id;
5169		- }
5170		- // An array node that represents one attachment doesn't contain other
5171		- // attachments inside it — done with this branch.
5172		- return;
5173		- }
5174		-
5175		- // Recurse: repeater rows, flexible-content layouts, groups, etc.
5176		- foreach ($value as $sub) {
5177		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($sub, $out, $depth + 1);
5178		- }
5179		- return;
5180		- }
5181		-
5182		- // Plain numeric attachment ID (ACF File field set to "Return: ID")
5183		- if (is_numeric($value)) {
5184		- $att_id = (int) $value;
5185		- if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5186		- $out[] = $att_id;
5187		- }
5188		- return;
5189		- }
5190		-
5191		- // Plain string — URL pointing at a PDF (ACF File field set to "Return: URL", or a custom URL/text field)
5192		- if (is_string($value)) {
5193		- $trimmed = trim($value);
5194		- if ($trimmed !== '' && $this->mxchat_url_looks_like_pdf($trimmed)) {
5195		- $att_id = (int) attachment_url_to_postid($trimmed);
5196		- if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5197		- $out[] = $att_id;
5198		- }
5199		- }
5200		- return;
5201		- }
5202		-}
5203		-
5204		-/**
5205		- * Heuristic: does this URL/string look like a PDF reference?
5206		- * Tolerates query strings and fragments (#page=2).
5207		- */
5208		-private function mxchat_url_looks_like_pdf($url) {
5209		- if (!is_string($url) || $url === '') {
5210		- return false;
5211		- }
5212		- // Strip query + fragment before checking extension
5213		- $path = preg_replace('/[?#].*$/', '', $url);
5214		- return (bool) preg_match('/\.pdf$/i', $path);
5215		-}
5216		-
5217		-/**
5218		- * Extract text from a PDF attachment by ID using the bundled Smalot parser.
5219		- * Reads the file directly from disk via get_attached_file (no HTTP fetch).
5220		- * Result is cached on the attachment as post_meta keyed by file mtime so we
5221		- * only parse the same PDF once unless the file changes on disk.
5222		- *
5223		- * @param int $attachment_id
5224		- * @return string Extracted plain text, or '' on failure.
5225		- */
5226		-private function mxchat_extract_pdf_text_by_attachment_id($attachment_id) {
5227		- $attachment_id = (int) $attachment_id;
5228		- if ($attachment_id <= 0) {
5229		- return '';
5230		- }
5231		- if (get_post_mime_type($attachment_id) !== 'application/pdf') {
5232		- return '';
5233		- }
5234		-
5235		- $pdf_path = get_attached_file($attachment_id);
5236		- if (empty($pdf_path) || !file_exists($pdf_path) || !is_readable($pdf_path)) {
5237		- return '';
5238		- }
5239		-
5240		- // Raw-file size cap. Parsing very large PDFs can OOM the request; skip with a log entry
5241		- // and let the rest of the ACF content land in the KB. Filterable for users who need it bigger.
5242		- $default_max_bytes = 25 * 1024 * 1024;
5243		- $max_bytes = (int) apply_filters('mxchat_acf_pdf_max_bytes', $default_max_bytes, $attachment_id, $pdf_path);
5244		- if ($max_bytes > 0) {
5245		- $file_size = @filesize($pdf_path);
5246		- if ($file_size !== false && $file_size > $max_bytes) {
5247		- error_log(sprintf(
5248		- '[mxchat] ACF PDF skipped (over size cap): attachment %d "%s" %d bytes > cap %d',
5249		- $attachment_id,
5250		- basename($pdf_path),
5251		- $file_size,
5252		- $max_bytes
5253		- ));
5254		- return '';
5255		- }
5256		- }
5257		-
5258		- $mtime = @filemtime($pdf_path);
5259		- $cache_meta_key = '_mxchat_acf_pdf_text_v1';
5260		- $cached = get_post_meta($attachment_id, $cache_meta_key, true);
5261		- if (is_array($cached) && isset($cached['mtime'], $cached['text']) && (int) $cached['mtime'] === (int) $mtime) {
5262		- return (string) $cached['text'];
5263		- }
5264		-
5265		- $text = '';
5266		- try {
5267		- if (function_exists('mxchat_load_pdf_parser')) {
5268		- mxchat_load_pdf_parser();
5269		- }
5270		- if (!class_exists('\\Smalot\\PdfParser\\Parser')) {
5271		- return '';
5272		- }
5273		- $parser = new \Smalot\PdfParser\Parser();
5274		- $pdf = $parser->parseFile($pdf_path);
5275		- $pages = $pdf->getPages();
5276		- $page_texts = array();
5277		- foreach ($pages as $page) {
5278		- $page_text = '';
5279		- try {
5280		- $page_text = $page->getText();
5281		- } catch (\Exception $e) {
5282		- $page_text = '';
5283		- }
5284		- if (!empty($page_text)) {
5285		- $page_texts[] = $page_text;
5286		- }
5287		- }
5288		- $text = trim(implode("\n\n", $page_texts));
5289		- } catch (\Exception $e) {
5290		- error_log('[mxchat] ACF PDF extraction failed for attachment ' . $attachment_id . ': ' . $e->getMessage());
5291		- return '';
5292		- } catch (\Throwable $e) {
5293		- error_log('[mxchat] ACF PDF extraction error for attachment ' . $attachment_id . ': ' . $e->getMessage());
5294		- return '';
5295		- }
5296		-
5297		- // Cap per-PDF text to avoid blowing up the embedding payload on enormous PDFs.
5298		- // The chunker downstream will still split this into multiple vectors.
5299		- $max_len = (int) apply_filters('mxchat_acf_pdf_text_max_length', 50000);
5300		- if ($max_len > 0 && strlen($text) > $max_len) {
5301		- $text = substr($text, 0, $max_len);
5302		- }
5303		-
5304		- update_post_meta($attachment_id, $cache_meta_key, array(
5305		- 'mtime' => (int) $mtime,
5306		- 'text' => $text,
5307		- ));
5308		-
5309		- return $text;
5310		-}
5311		-
5312		-/**
5313		- * Handle ACF save - fires after ACF fields are saved
5314		- * This ensures ACF field data is available when syncing to knowledge base
5315		- */
5316		-public function mxchat_handle_acf_save($post_id) {
5317		- // Skip if not a valid post
5318		- if (!$post_id || $post_id === 'options') {
5319		- return;
5320		- }
5321		-
5322		- // Skip autosaves and revisions
	2953	+public function mxchat_handle_post_update($post_id, $post, $update) {
	2954	+ // Basic validation checks
5323	2955	if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5324	2956	return;
5325	2957	}
5326		-
5327		- $post = get_post($post_id);
5328		- if (!$post) {
5329		- return;
5330		- }
5331		-
5332		- $post_type = $post->post_type;
5333		-
5334		- // Check if sync is enabled for this post type
5335		- $should_sync = false;
5336		-
5337		- if ($post_type === 'post' && get_option('mxchat_auto_sync_posts') === '1') {
5338		- $should_sync = true;
5339		- } else if ($post_type === 'page' && get_option('mxchat_auto_sync_pages') === '1') {
5340		- $should_sync = true;
5341		- } else if ($post_type === 'product' && class_exists('WooCommerce')) {
5342		- // WooCommerce products - check if WooCommerce integration is enabled
5343		- $options = get_option('mxchat_options', array());
5344		- if (isset($options['enable_woocommerce_integration']) &&
5345		- ($options['enable_woocommerce_integration'] === '1' || $options['enable_woocommerce_integration'] === 'on')) {
5346		- $should_sync = true;
5347		- }
5348		- } else {
5349		- // Check custom post types
5350		- $option_name = 'mxchat_auto_sync_' . $post_type;
5351		- if (get_option($option_name) === '1') {
5352		- $should_sync = true;
5353		- }
5354		- }
5355		-
5356		- if (!$should_sync) {
5357		- return;
5358		- }
5359		-
5360		- // Only process published posts
	2958	+
	2959	+ // Only process published content
5361	2960	if ($post->post_status !== 'publish') {
5362	2961	return;
5363	2962	}
5364		-
5365		- // Check if this post has any ACF fields - if not, no need to re-sync
5366		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
5367		- if (empty($acf_fields)) {
5368		- return;
5369		- }
5370		-
5371		- // Use a transient to prevent duplicate processing (post_updated may have already run)
5372		- $transient_key = 'mxchat_acf_synced_' . $post_id;
5373		- if (get_transient($transient_key)) {
5374		- return;
5375		- }
5376		- set_transient($transient_key, true, 60); // Prevent re-processing for 60 seconds
5377		-
5378		- // Re-run the sync with ACF data now available
5379		- // We pass $update=true since this is effectively an update with ACF data
5380		- $this->mxchat_handle_post_update($post_id, $post, true);
5381		-}
5382		-
5383		-public function mxchat_handle_post_update($post_id, $post, $update) {
5384		- // Basic validation checks
5385		- if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5386		- return;
5387		- }
5388	2963
5389	2964	$post_type = $post->post_type;
5390	2965
5391	2966	// Check if sync is enabled for this post type
		@@ -5407,276 +2982,80 @@
5407	2982	if (!$should_sync) {
5408	2983	return;
5409	2984	}
5410	2985
5411		- // Check if we have stored the previous status and URL in our transients
5412		- $previous_status_key = 'mxchat_prev_status_' . $post_id;
5413		- $previous_status = get_transient($previous_status_key);
	2986	+ // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
	2987	+ $title = get_the_title($post_id);
	2988	+ $content = get_post_field('post_content', $post_id);
5414	2989
5415		- $previous_url_key = 'mxchat_prev_url_' . $post_id;
5416		- $previous_url = get_transient($previous_url_key);
	2990	+ // Apply WordPress content filters to get properly formatted content
	2991	+ $content = apply_filters('the_content', $content);
5417	2992
5418		- // If the post was previously published but is now not published, remove from knowledge base
5419		- if ($previous_status === 'publish' && $post->post_status !== 'publish') {
5420		- // Use the stored URL from when it was published, or fall back to current permalink
5421		- $source_url = $previous_url ?: get_permalink($post_id);
5422		-
5423		- if ($source_url) {
5424		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
5425		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	2993	+ // Strip tags but preserve structure
	2994	+ $content = wp_strip_all_tags($content);
	2995	+
	2996	+ // Combine title and content
	2997	+ $final_content = $title . "\n\n" . $content;
	2998	+
	2999	+ // For custom post types like job_listing, include additional fields
	3000	+ if ($post_type === 'job_listing') {
	3001	+ // Add job-specific meta if available
	3002	+ $job_location = get_post_meta($post_id, '_job_location', true);
	3003	+ if (!empty($job_location)) {
	3004	+ $final_content .= "\n\nLocation: " . $job_location;
5426	3005	}
5427		-
5428		- // Clean up the transients and exit early
5429		- delete_transient($previous_status_key);
5430		- delete_transient($previous_url_key);
5431		- return;
5432		- }
5433		-
5434		- // Slug/permalink rename while still published: delete the old vectors before upserting new ones.
5435		- // Without this, md5(old_url) vectors (base + chunks) would be orphaned under the stale URL.
5436		- if ($post->post_status === 'publish' && !empty($previous_url)) {
5437		- $current_url = get_permalink($post_id);
5438		- if ($current_url && $current_url !== $previous_url) {
5439		- MxChat_Utils::delete_chunks_for_url($previous_url, 'default');
5440		- }
5441		- }
5442		-
5443		- // Store the current status for next time (if this is an update)
5444		- if ($update) {
5445		- set_transient($previous_status_key, $post->post_status, DAY_IN_SECONDS);
5446		-
5447		- // If the post is currently published, also store its URL
5448		- if ($post->post_status === 'publish') {
5449		- $current_url = get_permalink($post_id);
5450		- set_transient($previous_url_key, $current_url, DAY_IN_SECONDS);
5451		- }
5452		- }
5453		-
5454		- // Only process currently published content for adding/updating
5455		- if ($post->post_status === 'publish') {
5456		- // Get the source URL
5457		- $source_url = get_permalink($post_id);
5458	3006
5459		- // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
5460		- $title = get_the_title($post_id);
5461		- $content = get_post_field('post_content', $post_id);
5462		- $excerpt = get_post_field('post_excerpt', $post_id);
5463		-
5464		- // Remove shortcode tags but preserve content inside them
5465		- $content = $this->strip_shortcode_tags_preserve_content($content);
5466		- $excerpt = $this->strip_shortcode_tags_preserve_content($excerpt);
5467		-
5468		- // Strip tags but preserve structure (don't use 'the_content' filter as it may re-add shortcodes)
5469		- $content = wp_strip_all_tags($content);
5470		-
5471		- // Combine title, short description (if exists), and content
5472		- $final_content = $title . "\n\n";
5473		-
5474		- // Add short description if it exists (WooCommerce products use post_excerpt for short description)
5475		- if (!empty($excerpt)) {
5476		- $final_content .= "Short Description: " . wp_strip_all_tags($excerpt) . "\n\n";
5477		- }
5478		-
5479		- $final_content .= $content;
5480		-
5481		- // For WooCommerce products, include pricing and product details
5482		- if ($post_type === 'product' && class_exists('WooCommerce')) {
5483		- $product = wc_get_product($post_id);
5484		-
5485		- if ($product) {
5486		- // Get pricing information
5487		- $regular_price = $product->get_regular_price();
5488		- $sale_price = $product->get_sale_price();
5489		- $price = $product->get_price();
5490		- $sku = $product->get_sku();
5491		-
5492		- // Get currency symbol
5493		- $currency_symbol = get_woocommerce_currency_symbol();
5494		-
5495		- // Add pricing information
5496		- $final_content .= "\n";
5497		- if (!empty($regular_price)) {
5498		- $final_content .= "Price: " . $currency_symbol . $regular_price . "\n";
5499		- } elseif (!empty($price)) {
5500		- $final_content .= "Price: " . $currency_symbol . $price . "\n";
5501		- }
5502		-
5503		- if (!empty($sale_price) && $sale_price !== $regular_price) {
5504		- $final_content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
5505		- }
5506		-
5507		- // Handle variable products - show price range
5508		- if ($product->is_type('variable')) {
5509		- $min_price = $product->get_variation_price('min');
5510		- $max_price = $product->get_variation_price('max');
5511		- if ($min_price !== $max_price) {
5512		- $final_content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
5513		- }
5514		- }
5515		-
5516		- if (!empty($sku)) {
5517		- $final_content .= "SKU: " . $sku . "\n";
5518		- }
5519		-
5520		- // Get product categories
5521		- $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
5522		- if (!empty($categories) && !is_wp_error($categories)) {
5523		- $final_content .= "Categories: " . implode(', ', $categories) . "\n";
5524		- }
	3007	+ // Get job type terms
	3008	+ $job_types = get_the_terms($post_id, 'job_listing_type');
	3009	+ if (!empty($job_types) && !is_wp_error($job_types)) {
	3010	+ $types = array();
	3011	+ foreach ($job_types as $type) {
	3012	+ $types[] = $type->name;
5525	3013	}
	3014	+ $final_content .= "\n\nJob Type: " . implode(', ', $types);
5526	3015	}
5527		-
5528		- // For custom post types like job_listing, include additional fields
5529		- if ($post_type === 'job_listing') {
5530		- // Add job-specific meta if available
5531		- $job_location = get_post_meta($post_id, '_job_location', true);
5532		- if (!empty($job_location)) {
5533		- $final_content .= "\n\nLocation: " . $job_location;
5534		- }
5535		-
5536		- // Get job type terms
5537		- $job_types = get_the_terms($post_id, 'job_listing_type');
5538		- if (!empty($job_types) && !is_wp_error($job_types)) {
5539		- $types = array();
5540		- foreach ($job_types as $type) {
5541		- $types[] = $type->name;
5542		- }
5543		- $final_content .= "\n\nJob Type: " . implode(', ', $types);
5544		- }
5545		-
5546		- // Get company name if available
5547		- $company_name = get_post_meta($post_id, '_company_name', true);
5548		- if (!empty($company_name)) {
5549		- $final_content .= "\n\nCompany: " . $company_name;
5550		- }
5551		- }
5552		-
5553		- // ADD ACF FIELDS SUPPORT (matches ajax_mxchat_process_selected_content behavior)
5554		- $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
5555		- if (!empty($acf_fields)) {
5556		- $acf_content_parts = array();
5557		- $pdf_attachment_ids = array();
5558		-
5559		- foreach ($acf_fields as $field_name => $field_value) {
5560		- $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
5561		- if (!empty($formatted_value)) {
5562		- // Convert field name to readable label
5563		- $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name));
5564		- $acf_content_parts[] = $field_label . ": " . $formatted_value;
5565		- }
5566		-
5567		- $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
5568		- }
5569		-
5570		- if (!empty($acf_content_parts)) {
5571		- $final_content .= "\n\n" . implode("\n", $acf_content_parts);
5572		- }
5573		-
5574		- // Gate the auto-sync PDF-extraction loop behind an opt-in option.
5575		- // Mirrors the per-batch checkbox the manual content selector has; the
5576		- // 25 MB size cap lives in the shared extractor so it applies in both
5577		- // paths regardless. Default OFF — re-parsing every ACF PDF on every
5578		- // editor save is expensive and most sites don't want it.
5579		- $autosync_extract_acf_pdfs = get_option('mxchat_auto_sync_acf_pdfs', '0') === '1';
5580		- if ($autosync_extract_acf_pdfs && !empty($pdf_attachment_ids)) {
5581		- $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
5582		- $pdf_sections = array();
5583		- foreach ($pdf_attachment_ids as $att_id) {
5584		- $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
5585		- if (!empty($pdf_text)) {
5586		- $pdf_title = get_the_title($att_id);
5587		- $pdf_url = wp_get_attachment_url($att_id);
5588		- $header = 'PDF Attachment';
5589		- if (!empty($pdf_title)) {
5590		- $header .= ': ' . $pdf_title;
5591		- }
5592		- if (!empty($pdf_url)) {
5593		- $header .= ' (' . $pdf_url . ')';
5594		- }
5595		- $pdf_sections[] = $header . "\n" . $pdf_text;
5596		- }
5597		- }
5598		- if (!empty($pdf_sections)) {
5599		- $final_content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
5600		- }
5601		- }
5602		- }
5603		-
5604		- // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
5605		- $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
5606		- if (!empty($custom_meta)) {
5607		- $meta_content_parts = array();
5608		-
5609		- foreach ($custom_meta as $meta_key => $meta_value) {
5610		- // Convert meta key to readable label
5611		- $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
5612		- $meta_content_parts[] = $meta_label . ": " . $meta_value;
5613		- }
5614		-
5615		- if (!empty($meta_content_parts)) {
5616		- $final_content .= "\n\n" . implode("\n", $meta_content_parts);
5617		- }
5618		- }
5619		-
5620		- // Get API key with proper model detection
5621		- $options = get_option('mxchat_options');
5622		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
5623	3016
5624		- if (strpos($selected_model, 'voyage') === 0) {
5625		- $api_key = $options['voyage_api_key'] ?? '';
5626		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
5627		- $api_key = $options['gemini_api_key'] ?? '';
5628		- } else {
5629		- $api_key = $options['api_key'] ?? '';
	3017	+ // Get company name if available
	3018	+ $company_name = get_post_meta($post_id, '_company_name', true);
	3019	+ if (!empty($company_name)) {
	3020	+ $final_content .= "\n\nCompany: " . $company_name;
5630	3021	}
5631		-
5632		- if (empty($api_key)) {
5633		- return;
5634		- }
5635		-
5636		- // Use the centralized utility function for storage
5637		- $result = MxChat_Utils::submit_content_to_db(
5638		- $final_content,
5639		- $source_url,
5640		- $api_key,
5641		- md5($source_url) // Vector ID for Pinecone
5642		- );
5643		-
5644		- // After successful storage, apply role restriction based on tags
5645		- if (!is_wp_error($result)) {
5646		- $this->apply_role_restriction_to_post($post_id, $source_url);
5647		- }
5648	3022	}
5649	3023
5650		- // Clean up the stored previous status if not used above
5651		- if ($previous_status !== 'publish' || $post->post_status === 'publish') {
5652		- delete_transient($previous_status_key);
5653		- delete_transient($previous_url_key);
	3024	+ // Get the source URL
	3025	+ $source_url = get_permalink($post_id);
	3026	+
	3027	+ // Get API key with proper model detection
	3028	+ $options = get_option('mxchat_options');
	3029	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	3030	+
	3031	+ if (strpos($selected_model, 'voyage') === 0) {
	3032	+ $api_key = $options['voyage_api_key'] ?? '';
	3033	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	3034	+ $api_key = $options['gemini_api_key'] ?? '';
	3035	+ } else {
	3036	+ $api_key = $options['api_key'] ?? '';
5654	3037	}
5655		-}
5656		-
5657		-/**
5658		- * Store the post status and URL before update to detect status transitions
5659		- * This runs before the post is actually updated in the database
5660		- */
5661		-public function mxchat_store_pre_update_status($post_id, $data) {
5662		- // Get the current post from database (before update)
5663		- $current_post = get_post($post_id);
5664	3038
5665		- if ($current_post) {
5666		- // Store the current status temporarily
5667		- $status_key = 'mxchat_prev_status_' . $post_id;
5668		- set_transient($status_key, $current_post->post_status, HOUR_IN_SECONDS);
5669		-
5670		- // If the post is currently published, also store its URL
5671		- if ($current_post->post_status === 'publish') {
5672		- $url_key = 'mxchat_prev_url_' . $post_id;
5673		- $current_url = get_permalink($post_id);
5674		- set_transient($url_key, $current_url, HOUR_IN_SECONDS);
5675		- }
	3039	+ if (empty($api_key)) {
	3040	+ error_log('MxChat Auto-sync: No API key configured for embedding model');
	3041	+ return;
5676	3042	}
	3043	+
	3044	+ // Use the centralized utility function for storage
	3045	+ $result = MxChat_Utils::submit_content_to_db(
	3046	+ $final_content,
	3047	+ $source_url,
	3048	+ $api_key,
	3049	+ md5($source_url) // Vector ID for Pinecone
	3050	+ );
	3051	+
	3052	+ if (is_wp_error($result)) {
	3053	+ error_log('MxChat Auto-sync failed for post ' . $post_id . ': ' . $result->get_error_message());
	3054	+ }
5677	3055	}
5678	3056
	3057	+
5679	3058	public function mxchat_handle_post_delete($post_id) {
5680	3059	// Get post data before it's deleted
5681	3060	$post = get_post($post_id);
5682	3061
		@@ -5706,51 +3085,85 @@
5706	3085	if (!$should_sync) {
5707	3086	return;
5708	3087	}
5709	3088
5710		- // Resolve the pre-trash URL. wp_trash_post renames the slug with "__trashed" before firing
5711		- // this hook, so get_permalink() here would return the trashed URL and md5() would miss the
5712		- // real vector IDs stored under the original URL.
5713		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
	3089	+ // Get the URL before post is deleted
	3090	+ $source_url = get_permalink($post_id);
5714	3091	if (!$source_url) {
5715		- //error_log('MXChat: Failed to resolve source URL for post ' . $post_id);
	3092	+ error_log('MXChat: Failed to get permalink for post ' . $post_id);
5716	3093	return;
5717	3094	}
5718	3095
5719		- // Use chunk-aware deletion (handles both chunked and non-chunked content)
5720		- $delete_result = MxChat_Utils::delete_chunks_for_url($source_url, 'default');
	3096	+ // Check if Pinecone is enabled
	3097	+ $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
	3098	+ $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5721	3099
5722		- if (is_wp_error($delete_result)) {
5723		- //error_log('MXChat: Chunk-aware deletion failed for URL: ' . $source_url . ' - ' . $delete_result->get_error_message());
	3100	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3101	+ // Delete from Pinecone
	3102	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
	3103	+ } else {
	3104	+ // Delete from WordPress DB
	3105	+ global $wpdb;
	3106	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	3107	+
	3108	+ $result = $wpdb->delete(
	3109	+ $table_name,
	3110	+ array('source_url' => $source_url),
	3111	+ array('%s')
	3112	+ );
	3113	+
	3114	+ if ($result === false) {
	3115	+ error_log('MXChat: WordPress DB deletion failed for URL: ' . $source_url);
	3116	+ }
5724	3117	}
	3118	+}
5725	3119
5726		- delete_transient('mxchat_prev_url_' . $post_id);
5727		- delete_transient('mxchat_prev_status_' . $post_id);
5728		-}
5729	3120
5730		-/**
5731		- * Resolve the source URL for a post being trashed/deleted.
5732		- *
5733		- * Why: wp_trash_post appends "__trashed" to the slug before the wp_trash_post action fires, so
5734		- * get_permalink() returns a URL whose md5() won't match the vector IDs stored in Pinecone or
5735		- * the source_url rows in the WP DB. Prefer the URL captured by mxchat_store_pre_update_status
5736		- * (runs on pre_post_update, before the rename); fall back to stripping the __trashed suffix.
5737		- */
5738		-private function mxchat_resolve_pre_trash_url($post_id) {
5739		- $previous_url = get_transient('mxchat_prev_url_' . $post_id);
5740		- if (!empty($previous_url)) {
5741		- return $previous_url;
5742		- }
	3121	+ /**
	3122	+ * Deletes data from Pinecone using a source URL
	3123	+ */
	3124	+ public function mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options) {
	3125	+ $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
	3126	+ $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
5743	3127
5744		- $current = get_permalink($post_id);
5745		- if (!$current) {
5746		- return '';
5747		- }
5748		- return preg_replace('#__trashed(/?)$#', '$1', $current);
5749		-}
	3128	+ if (empty($host) || empty($api_key)) {
	3129	+ //error_log('MXChat: Pinecone deletion failed - missing configuration');
	3130	+ return false;
	3131	+ }
5750	3132
	3133	+ $api_endpoint = "https://{$host}/vectors/delete";
	3134	+ $vector_id = md5($source_url);
5751	3135
	3136	+ $request_body = array(
	3137	+ 'ids' => array($vector_id)
	3138	+ );
5752	3139
	3140	+ $response = wp_remote_post($api_endpoint, array(
	3141	+ 'headers' => array(
	3142	+ 'Api-Key' => $api_key,
	3143	+ 'accept' => 'application/json',
	3144	+ 'content-type' => 'application/json'
	3145	+ ),
	3146	+ 'body' => wp_json_encode($request_body),
	3147	+ 'timeout' => 30
	3148	+ ));
	3149	+
	3150	+ if (is_wp_error($response)) {
	3151	+ //error_log('MXChat: Pinecone deletion error - ' . $response->get_error_message());
	3152	+ return false;
	3153	+ }
	3154	+
	3155	+ $response_code = wp_remote_retrieve_response_code($response);
	3156	+ if ($response_code !== 200) {
	3157	+ //error_log('MXChat: Pinecone deletion failed with status ' . $response_code);
	3158	+ return false;
	3159	+ }
	3160	+
	3161	+ return true;
	3162	+ }
	3163	+
	3164	+
	3165	+
5753	3166	public function mxchat_handle_product_change($post_id, $post, $update) {
5754	3167	if ($post->post_type !== 'product') {
5755	3168	return;
5756	3169	}
		@@ -5772,12 +3185,11 @@
5772	3185	if (!isset($this->options['enable_woocommerce_integration']) ||
5773	3186	!in_array($this->options['enable_woocommerce_integration'], ['1', 'on'])) {
5774	3187	return;
5775	3188	}
5776		-
	3189	+
5777	3190	$source_url = get_permalink($product->get_id());
5778		- $product_id = $product->get_id();
5779		-
	3191	+
5780	3192	// Build product content
5781	3193	$title = $product->get_name();
5782	3194	$description = $product->get_description();
5783	3195	$short_description = $product->get_short_description();
		@@ -5782,92 +3194,35 @@
5782	3194	$description = $product->get_description();
5783	3195	$short_description = $product->get_short_description();
5784	3196	$regular_price = $product->get_regular_price();
5785	3197	$sale_price = $product->get_sale_price();
5786		- $price = $product->get_price();
5787	3198	$sku = $product->get_sku();
5788		-
5789		- // Get currency symbol
5790		- $currency_symbol = get_woocommerce_currency_symbol();
5791		-
	3199	+
5792	3200	// Format content consistently
5793	3201	$content = $title . "\n\n";
5794		-
5795		- if (!empty($short_description)) {
5796		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
5797		- }
5798		-
	3202	+
5799	3203	if (!empty($description)) {
5800	3204	$content .= wp_strip_all_tags($description) . "\n\n";
5801	3205	}
5802		-
5803		- // Add pricing information
5804		- if (!empty($regular_price)) {
5805		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
5806		- } elseif (!empty($price)) {
5807		- $content .= "Price: " . $currency_symbol . $price . "\n";
	3206	+
	3207	+ if (!empty($short_description)) {
	3208	+ $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
5808	3209	}
5809		-
5810		- if (!empty($sale_price) && $sale_price !== $regular_price) {
5811		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
	3210	+
	3211	+ $content .= "Price: $" . $regular_price . "\n";
	3212	+
	3213	+ if (!empty($sale_price)) {
	3214	+ $content .= "Sale Price: $" . $sale_price . "\n";
5812	3215	}
5813		-
5814		- // Handle variable products - show price range
5815		- if ($product->is_type('variable')) {
5816		- $min_price = $product->get_variation_price('min');
5817		- $max_price = $product->get_variation_price('max');
5818		- if ($min_price !== $max_price) {
5819		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
5820		- }
5821		- }
5822		-
	3216	+
5823	3217	if (!empty($sku)) {
5824	3218	$content .= "SKU: " . $sku . "\n";
5825	3219	}
5826		-
5827		- // Get product categories
5828		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
5829		- if (!empty($categories) && !is_wp_error($categories)) {
5830		- $content .= "Categories: " . implode(', ', $categories) . "\n";
5831		- }
5832		-
5833		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
5834		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
5835		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
5836		- foreach ($custom_tabs as $tab) {
5837		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
5838		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
5839		-
5840		- if (!empty($tab_title) && !empty($tab_content)) {
5841		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
5842		- }
5843		- }
5844		- }
5845		-
5846		- // Also check for reusable/saved tabs applied to this product
5847		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
5848		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
5849		- // Get the saved tabs option
5850		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
5851		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
5852		- foreach ($applied_saved_tabs as $saved_tab_id) {
5853		- if (isset($saved_tabs[$saved_tab_id])) {
5854		- $tab = $saved_tabs[$saved_tab_id];
5855		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
5856		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
5857		-
5858		- if (!empty($tab_title) && !empty($tab_content)) {
5859		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
5860		- }
5861		- }
5862		- }
5863		- }
5864		- }
5865		-
	3220	+
5866	3221	// Get API key with proper model detection
5867	3222	$options = get_option('mxchat_options');
5868	3223	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
5869		-
	3224	+
5870	3225	if (strpos($selected_model, 'voyage') === 0) {
5871	3226	$api_key = $options['voyage_api_key'] ?? '';
5872	3227	} elseif (strpos($selected_model, 'gemini-embedding') === 0) {
5873	3228	$api_key = $options['gemini_api_key'] ?? '';
		@@ -5873,14 +3228,14 @@
5873	3228	$api_key = $options['gemini_api_key'] ?? '';
5874	3229	} else {
5875	3230	$api_key = $options['api_key'] ?? '';
5876	3231	}
5877		-
	3232	+
5878	3233	if (empty($api_key)) {
5879		- //error_log('MxChat Auto-sync: No API key configured for embedding model');
	3234	+ error_log('MxChat Auto-sync: No API key configured for embedding model');
5880	3235	return;
5881	3236	}
5882		-
	3237	+
5883	3238	// Use the centralized utility function for storage
5884	3239	$result = MxChat_Utils::submit_content_to_db(
5885	3240	$content,
5886	3241	$source_url,
		@@ -5886,16 +3241,11 @@
5886	3241	$source_url,
5887	3242	$api_key,
5888	3243	md5($source_url) // Vector ID for Pinecone
5889	3244	);
5890		-
5891		- // After successful storage, apply role restriction based on tags
5892		- if (!is_wp_error($result)) {
5893		- $this->apply_role_restriction_to_post($product_id, $source_url);
5894		- }
5895		-
	3245	+
5896	3246	if (is_wp_error($result)) {
5897		- //error_log('MxChat WooCommerce sync failed for product ' . $product_id . ': ' . $result->get_error_message());
	3247	+ error_log('MxChat WooCommerce sync failed for product ' . $product->get_id() . ': ' . $result->get_error_message());
5898	3248	}
5899	3249	}
5900	3250
5901	3251	public function mxchat_handle_product_delete($post_id) {
		@@ -5902,1112 +3252,30 @@
5902	3252	if (get_post_type($post_id) !== 'product') {
5903	3253	return;
5904	3254	}
5905	3255
5906		- $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
5907		- if (!$source_url) {
5908		- return;
5909		- }
	3256	+ $source_url = get_permalink($post_id);
5910	3257
5911		- // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
5912		- MxChat_Utils::delete_chunks_for_url($source_url, 'default');
5913		-
5914		- delete_transient('mxchat_prev_url_' . $post_id);
5915		- delete_transient('mxchat_prev_status_' . $post_id);
5916		-}
5917		-
5918		-/**
5919		- * Handle individual Pinecone content deletion
5920		- */
5921		-public function mxchat_handle_pinecone_prompt_delete() {
5922		- // Check permissions
5923		- if (!current_user_can('manage_options')) {
5924		- wp_die(esc_html__('You do not have sufficient permissions.', 'mxchat'));
5925		- }
5926		-
5927		- // Verify nonce
5928		- if (!isset($_GET['_wpnonce']) || !wp_verify_nonce($_GET['_wpnonce'], 'mxchat_delete_pinecone_prompt_nonce')) {
5929		- wp_die(esc_html__('Security check failed.', 'mxchat'));
5930		- }
5931		-
5932		- $vector_id = isset($_GET['vector_id']) ? sanitize_text_field($_GET['vector_id']) : '';
5933		-
5934		- if (empty($vector_id)) {
5935		- set_transient('mxchat_admin_notice_error',
5936		- esc_html__('Invalid vector ID.', 'mxchat'),
5937		- 30
5938		- );
5939		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
5940		- exit;
5941		- }
5942		-
5943		- // Get Pinecone settings
	3258	+ // Check if Pinecone is enabled
5944	3259	$pinecone_options = get_option('mxchat_pinecone_addon_options', array());
5945	3260	$use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
5946		-
5947		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
5948		- set_transient('mxchat_admin_notice_error',
5949		- esc_html__('Pinecone is not properly configured.', 'mxchat'),
5950		- 30
5951		- );
5952		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
5953		- exit;
5954		- }
5955		-
5956		- // Delete from Pinecone
5957		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
5958		- $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
5959		- $vector_id,
5960		- $pinecone_options['mxchat_pinecone_api_key'],
5961		- $pinecone_options['mxchat_pinecone_host']
5962		- );
5963		-
5964		- if ($result['success']) {
5965		- // No cache clearing needed since we removed caching
5966		- set_transient('mxchat_admin_notice_success',
5967		- esc_html__('Entry deleted successfully from Pinecone.', 'mxchat'),
5968		- 30
5969		- );
5970		- } else {
5971		- set_transient('mxchat_admin_notice_error',
5972		- esc_html__('Failed to delete entry: ', 'mxchat') . $result['message'],
5973		- 30
5974		- );
5975		- }
5976		-
5977		- wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
5978		- exit;
5979		-}
5980		-/**
5981		- * Handle individual Pinecone content deletion via AJAX
5982		- */
5983		-public function ajax_mxchat_delete_pinecone_prompt() {
5984		- // Verify nonce and permissions
5985		- if (!check_ajax_referer('mxchat_delete_pinecone_prompt_nonce', 'nonce', false)) {
5986		- wp_send_json_error('Invalid nonce');
5987		- exit;
5988		- }
5989		-
5990		- if (!current_user_can('manage_options')) {
5991		- wp_send_json_error('Unauthorized access');
5992		- exit;
5993		- }
5994		-
5995		- $vector_id = isset($_POST['vector_id']) ? sanitize_text_field($_POST['vector_id']) : '';
5996		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
5997		-
5998		- if (empty($vector_id)) {
5999		- wp_send_json_error('Missing vector ID');
6000		- exit;
6001		- }
6002		-
6003		- // Get bot-specific Pinecone settings
6004		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6005		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6006		-
6007		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6008		-
6009		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
6010		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
6011		- exit;
6012		- }
6013		-
6014		- // Delete from the correct Pinecone index
6015		- $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
6016		- $vector_id,
6017		- $pinecone_options['mxchat_pinecone_api_key'],
6018		- $pinecone_options['mxchat_pinecone_host']
6019		- );
6020		-
6021		- if ($result['success']) {
6022		- // No cache clearing needed since we removed caching
6023		- wp_send_json_success(array(
6024		- 'message' => 'Entry deleted successfully from Pinecone',
6025		- 'vector_id' => $vector_id,
6026		- 'bot_id' => $bot_id
6027		- ));
6028		- } else {
6029		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete from Pinecone: ' . $result['message'], array('vector_id' => $vector_id, 'bot_id' => $bot_id));
6030		- wp_send_json_error('Failed to delete from Pinecone: ' . $result['message']);
6031		- }
6032		-
6033		- exit;
6034		-}
6035	3261
6036		-/**
6037		- * Handle deletion of all chunks for a given source URL via AJAX
6038		- * Follows the same pattern as ajax_mxchat_delete_pinecone_prompt
6039		- */
6040		-public function ajax_mxchat_delete_chunks_by_url() {
6041		- // Verify nonce and permissions
6042		- if (!check_ajax_referer('mxchat_delete_chunks_nonce', 'nonce', false)) {
6043		- wp_send_json_error('Invalid nonce');
6044		- exit;
6045		- }
6046		-
6047		- if (!current_user_can('manage_options')) {
6048		- wp_send_json_error('Unauthorized access');
6049		- exit;
6050		- }
6051		-
6052		- $source_url = isset($_POST['source_url']) ? esc_url_raw($_POST['source_url']) : '';
6053		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
6054		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6055		-
6056		- if (empty($source_url)) {
6057		- wp_send_json_error('Missing source URL');
6058		- exit;
6059		- }
6060		-
6061		- // Generate the base vector ID from the source URL (same as how chunks are created)
6062		- $base_vector_id = md5($source_url);
6063		-
6064		- if ($data_source === 'pinecone') {
6065		- // Get bot-specific Pinecone settings (same as working delete function)
6066		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6067		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6068		-
6069		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6070		-
6071		- if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
6072		- wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
6073		- exit;
6074		- }
6075		-
6076		- $api_key = $pinecone_options['mxchat_pinecone_api_key'];
6077		- $host = $pinecone_options['mxchat_pinecone_host'];
6078		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
6079		-
6080		- // Collect all vector IDs to delete
6081		- $vectors_to_delete = array();
6082		-
6083		- // Add the original single-vector ID (for non-chunked content)
6084		- $vectors_to_delete[] = $base_vector_id;
6085		-
6086		- // Use Pinecone list API to find all chunk vectors with this prefix
6087		- // NOTE: Pinecone List API is a GET request with query parameters, not POST
6088		- $prefix = $base_vector_id . '_chunk_';
6089		-
6090		- $query_params = array(
6091		- 'prefix' => $prefix,
6092		- 'limit' => 100
6093		- );
6094		-
6095		- if (!empty($namespace)) {
6096		- $query_params['namespace'] = $namespace;
6097		- }
6098		-
6099		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
6100		-
6101		- $list_response = wp_remote_get($list_url, array(
6102		- 'headers' => array(
6103		- 'Api-Key' => $api_key,
6104		- 'accept' => 'application/json'
6105		- ),
6106		- 'timeout' => 30
6107		- ));
6108		-
6109		- if (!is_wp_error($list_response)) {
6110		- $list_body_response = wp_remote_retrieve_body($list_response);
6111		- $list_data = json_decode($list_body_response, true);
6112		- if (!empty($list_data['vectors'])) {
6113		- foreach ($list_data['vectors'] as $vector) {
6114		- if (isset($vector['id'])) {
6115		- $vectors_to_delete[] = $vector['id'];
6116		- }
6117		- }
6118		- }
6119		- }
6120		-
6121		- if (empty($vectors_to_delete)) {
6122		- wp_send_json_success(array(
6123		- 'message' => 'No vectors found to delete',
6124		- 'source_url' => $source_url
6125		- ));
6126		- exit;
6127		- }
6128		-
6129		- // Delete all vectors using the same endpoint as the working function
6130		- $delete_url = "https://{$host}/vectors/delete";
6131		-
6132		- $delete_body = array(
6133		- 'ids' => $vectors_to_delete
6134		- );
6135		-
6136		- if (!empty($namespace)) {
6137		- $delete_body['namespace'] = $namespace;
6138		- }
6139		-
6140		- $delete_response = wp_remote_post($delete_url, array(
6141		- 'headers' => array(
6142		- 'Api-Key' => $api_key,
6143		- 'accept' => 'application/json',
6144		- 'content-type' => 'application/json'
6145		- ),
6146		- 'body' => wp_json_encode($delete_body),
6147		- 'timeout' => 30
6148		- ));
6149		-
6150		- if (is_wp_error($delete_response)) {
6151		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete chunks from Pinecone: ' . $delete_response->get_error_message(), array('source_url' => $source_url));
6152		- wp_send_json_error('Failed to delete from Pinecone: ' . $delete_response->get_error_message());
6153		- exit;
6154		- }
6155		-
6156		- $response_code = wp_remote_retrieve_response_code($delete_response);
6157		-
6158		- if ($response_code !== 200) {
6159		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Pinecone API error (HTTP ' . $response_code . ')', array('source_url' => $source_url));
6160		- wp_send_json_error('Pinecone API error (HTTP ' . $response_code . ')');
6161		- exit;
6162		- }
6163		-
6164		- wp_send_json_success(array(
6165		- 'message' => 'All chunks deleted successfully from Pinecone',
6166		- 'source_url' => $source_url,
6167		- 'deleted_count' => count($vectors_to_delete)
6168		- ));
6169		-
	3262	+ if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
	3263	+ // Delete from Pinecone
	3264	+ $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
6170	3265	} else {
6171		- // WordPress database deletion
	3266	+ // Delete from WordPress DB
6172	3267	global $wpdb;
6173	3268	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6174		-
6175		- $result = $wpdb->delete(
6176		- $table_name,
6177		- array('source_url' => $source_url),
6178		- array('%s')
6179		- );
6180		-
6181		- if ($result === false) {
6182		- MxChat_Admin::mxchat_log_debug('database_error', 'Failed to delete from database: ' . $wpdb->last_error, array('source_url' => $source_url));
6183		- wp_send_json_error('Failed to delete from database: ' . $wpdb->last_error);
6184		- exit;
6185		- }
6186		-
6187		- wp_send_json_success(array(
6188		- 'message' => 'All chunks deleted successfully from database',
6189		- 'source_url' => $source_url,
6190		- 'deleted_count' => $result
6191		- ));
6192		- }
6193		-
6194		- exit;
6195		-}
6196		-
6197		-/**
6198		- * Handle individual WordPress database content deletion via AJAX
6199		- * Mirrors the Pinecone delete handler but for WordPress database entries
6200		- */
6201		-public function ajax_mxchat_delete_wordpress_prompt() {
6202		- // Verify nonce and permissions
6203		- if (!check_ajax_referer('mxchat_delete_wordpress_prompt_nonce', 'nonce', false)) {
6204		- wp_send_json_error('Invalid nonce');
6205		- exit;
6206		- }
6207		-
6208		- if (!current_user_can('manage_options')) {
6209		- wp_send_json_error('Unauthorized access');
6210		- exit;
6211		- }
6212		-
6213		- $entry_id = isset($_POST['entry_id']) ? intval($_POST['entry_id']) : 0;
6214		-
6215		- if (empty($entry_id)) {
6216		- wp_send_json_error('Missing entry ID');
6217		- exit;
6218		- }
6219		-
6220		- global $wpdb;
6221		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6222		-
6223		- // Clear cache for this entry
6224		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
6225		-
6226		- // Delete from database
6227		- $result = $wpdb->delete(
6228		- $table_name,
6229		- array('id' => $entry_id),
6230		- array('%d')
6231		- );
6232		-
6233		- if ($result !== false) {
6234		- wp_send_json_success(array(
6235		- 'message' => 'Entry deleted successfully',
6236		- 'entry_id' => $entry_id
6237		- ));
6238		- } else {
6239		- wp_send_json_error('Failed to delete entry from database');
6240		- }
6241		-
6242		- exit;
6243		-}
6244		-
6245		-/**
6246		- * Handle bulk deletion of knowledge entries via AJAX
6247		- * Supports both Pinecone and WordPress database entries
6248		- */
6249		-public function ajax_mxchat_bulk_delete_knowledge() {
6250		- // Verify nonce and permissions
6251		- if (!check_ajax_referer('mxchat_bulk_delete_knowledge_nonce', 'nonce', false)) {
6252		- wp_send_json_error('Invalid nonce');
6253		- exit;
6254		- }
6255		-
6256		- if (!current_user_can('manage_options')) {
6257		- wp_send_json_error('Unauthorized access');
6258		- exit;
6259		- }
6260		-
6261		- $entries = isset($_POST['entries']) ? $_POST['entries'] : array();
6262		- $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6263		-
6264		- if (empty($entries) || !is_array($entries)) {
6265		- wp_send_json_error('No entries provided');
6266		- exit;
6267		- }
6268		-
6269		- // Extend execution time — bulk Pinecone operations can take a while
6270		- if (function_exists('set_time_limit')) {
6271		- set_time_limit(120);
6272		- }
6273		-
6274		- $success_ids = array();
6275		- $failed_ids = array();
6276		- $errors = array();
6277		-
6278		- global $wpdb;
6279		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6280		-
6281		- // Get Pinecone manager for Pinecone deletions
6282		- $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6283		- $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6284		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6285		-
6286		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
6287		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
6288		-
6289		- // =============================================
6290		- // PHASE 1: Collect all Pinecone vector IDs
6291		- // and separate WordPress entries
6292		- // =============================================
6293		- $pinecone_entry_ids = array(); // entry IDs that are pinecone-sourced
6294		- $wordpress_entries = array(); // entries for WordPress DB deletion
6295		- $all_vector_ids = array(); // all pinecone vector IDs to delete in one batch
6296		-
6297		- foreach ($entries as $entry) {
6298		- $entry_id = sanitize_text_field($entry['id'] ?? '');
6299		- $source = sanitize_text_field($entry['source'] ?? 'wordpress');
6300		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
6301		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
6302		-
6303		- if (empty($entry_id)) {
6304		- continue;
6305		- }
6306		-
6307		- if ($source === 'pinecone') {
6308		- if (!$use_pinecone || empty($api_key)) {
6309		- $failed_ids[] = $entry_id;
6310		- $errors[] = "Pinecone not configured for entry: $entry_id";
6311		- continue;
6312		- }
6313		-
6314		- $pinecone_entry_ids[] = $entry_id;
6315		-
6316		- if ($is_group && !empty($source_url)) {
6317		- // Grouped/chunked entry: collect base ID + chunk IDs via List API
6318		- $base_vector_id = md5($source_url);
6319		- $all_vector_ids[] = $base_vector_id;
6320		-
6321		- $list_url = 'https://' . $host . '/vectors/list?prefix=' . urlencode($base_vector_id . '_chunk_') . '&limit=100';
6322		- $list_response = wp_remote_get($list_url, array(
6323		- 'headers' => array(
6324		- 'Api-Key' => $api_key,
6325		- 'accept' => 'application/json'
6326		- ),
6327		- 'timeout' => 30
6328		- ));
6329		-
6330		- if (!is_wp_error($list_response)) {
6331		- $list_body = json_decode(wp_remote_retrieve_body($list_response), true);
6332		- if (!empty($list_body['vectors']) && is_array($list_body['vectors'])) {
6333		- foreach ($list_body['vectors'] as $vector) {
6334		- if (isset($vector['id'])) {
6335		- $all_vector_ids[] = $vector['id'];
6336		- }
6337		- }
6338		- }
6339		- }
6340		- } else {
6341		- // Single entry: the entry_id IS the vector ID
6342		- $all_vector_ids[] = $entry_id;
6343		- }
6344		- } else {
6345		- $wordpress_entries[] = $entry;
6346		- }
6347		- }
6348		-
6349		- // =============================================
6350		- // PHASE 2: Single batch delete to Pinecone
6351		- // =============================================
6352		- if (!empty($all_vector_ids)) {
6353		- $all_vector_ids = array_values(array_unique($all_vector_ids));
6354		- $pinecone_success = true;
6355		- $batches = array_chunk($all_vector_ids, 100);
6356		-
6357		- foreach ($batches as $batch) {
6358		- $delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
6359		- 'headers' => array(
6360		- 'Api-Key' => $api_key,
6361		- 'accept' => 'application/json',
6362		- 'content-type' => 'application/json'
6363		- ),
6364		- 'body' => wp_json_encode(array('ids' => $batch)),
6365		- 'timeout' => 60
6366		- ));
6367		-
6368		- if (is_wp_error($delete_response)) {
6369		- $pinecone_success = false;
6370		- $errors[] = 'Pinecone batch deletion failed: ' . $delete_response->get_error_message();
6371		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed: ' . $delete_response->get_error_message());
6372		- } else {
6373		- $response_code = wp_remote_retrieve_response_code($delete_response);
6374		- if ($response_code !== 200) {
6375		- $pinecone_success = false;
6376		- $response_body = wp_remote_retrieve_body($delete_response);
6377		- $errors[] = "Pinecone API error (HTTP $response_code)";
6378		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed (HTTP ' . $response_code . ')', array('response' => substr($response_body, 0, 200)));
6379		- }
6380		- }
6381		- }
6382		-
6383		- // Mark all pinecone entries based on batch result
6384		- foreach ($pinecone_entry_ids as $eid) {
6385		- if ($pinecone_success) {
6386		- $success_ids[] = $eid;
6387		- } else {
6388		- $failed_ids[] = $eid;
6389		- }
6390		- }
6391		- }
6392		-
6393		- // =============================================
6394		- // PHASE 3: WordPress database deletions
6395		- // =============================================
6396		- foreach ($wordpress_entries as $entry) {
6397		- $entry_id = sanitize_text_field($entry['id'] ?? '');
6398		- $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
6399		- $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
6400		-
6401		- if (empty($entry_id)) {
6402		- continue;
6403		- }
6404		-
6405		- try {
6406		- if ($is_group && !empty($source_url)) {
6407		- $result = $wpdb->delete(
6408		- $table_name,
6409		- array('source_url' => $source_url),
6410		- array('%s')
6411		- );
6412		- } else {
6413		- wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
6414		- $result = $wpdb->delete(
6415		- $table_name,
6416		- array('id' => intval($entry_id)),
6417		- array('%d')
6418		- );
6419		- }
6420		-
6421		- if ($result !== false) {
6422		- $success_ids[] = $entry_id;
6423		- } else {
6424		- $failed_ids[] = $entry_id;
6425		- $errors[] = "Database error for entry: $entry_id";
6426		- }
6427		- } catch (Exception $e) {
6428		- $failed_ids[] = $entry_id;
6429		- $errors[] = $e->getMessage();
6430		- }
6431		- }
6432		-
6433		- wp_send_json_success(array(
6434		- 'success_ids' => $success_ids,
6435		- 'failed_ids' => $failed_ids,
6436		- 'errors' => $errors,
6437		- 'total_processed' => count($success_ids) + count($failed_ids)
6438		- ));
6439		-
6440		- exit;
6441		-}
6442		-
6443		-/**
6444		- * Get hierarchical roles for dropdown
6445		- */
6446		-public function mxchat_get_role_options() {
6447		- return array(
6448		- 'public' => __('Public (Everyone)', 'mxchat'),
6449		- 'logged_in' => __('Logged In Users', 'mxchat'),
6450		- 'subscriber' => __('Subscribers & Above', 'mxchat'),
6451		- 'contributor' => __('Contributors & Above', 'mxchat'),
6452		- 'author' => __('Authors & Above', 'mxchat'),
6453		- 'editor' => __('Editors & Above', 'mxchat'),
6454		- 'administrator' => __('Administrators Only', 'mxchat')
6455		- );
6456		-}
6457		-
6458		-/**
6459		- * Check if user has access to content based on role restriction
6460		- */
6461		-public function mxchat_user_has_content_access($role_restriction) {
6462		- // Public content is always accessible
6463		- if ($role_restriction === 'public' || empty($role_restriction)) {
6464		- return true;
6465		- }
6466		-
6467		- // Check if user is logged in for logged_in restriction
6468		- if ($role_restriction === 'logged_in') {
6469		- return is_user_logged_in();
6470		- }
6471		-
6472		- // If not logged in, no access to role-restricted content
6473		- if (!is_user_logged_in()) {
6474		- return false;
6475		- }
6476		-
6477		- $user = wp_get_current_user();
6478		- $user_roles = $user->roles;
6479		-
6480		- if (empty($user_roles)) {
6481		- return false;
6482		- }
6483		-
6484		- // Define role hierarchy (higher number = higher access)
6485		- $hierarchy = array(
6486		- 'subscriber' => 1,
6487		- 'contributor' => 2,
6488		- 'author' => 3,
6489		- 'editor' => 4,
6490		- 'administrator' => 5
6491		- );
6492		-
6493		- // Get required level
6494		- $required_level = isset($hierarchy[$role_restriction]) ? $hierarchy[$role_restriction] : 0;
6495		-
6496		- // Check if user has required level or higher
6497		- foreach ($user_roles as $user_role) {
6498		- $user_level = isset($hierarchy[$user_role]) ? $hierarchy[$user_role] : 0;
6499		- if ($user_level >= $required_level) {
6500		- return true;
6501		- }
6502		- }
6503		-
6504		- return false;
6505		-}
6506		-
6507		-/**
6508		- * Handle role restriction updates via AJAX
6509		- * Removed cache clearing call since we removed caching
6510		- */
6511		-public function ajax_mxchat_update_role_restriction() {
6512		- // Verify nonce and permissions
6513		- if (!check_ajax_referer('mxchat_update_role_nonce', 'nonce', false)) {
6514		- wp_send_json_error('Invalid nonce');
6515		- exit;
6516		- }
6517		-
6518		- if (!current_user_can('manage_options')) {
6519		- wp_send_json_error('Unauthorized access');
6520		- exit;
6521		- }
6522		-
6523		- $entry_id = isset($_POST['entry_id']) ? sanitize_text_field($_POST['entry_id']) : '';
6524		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
6525		- $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
6526		-
6527		- if (empty($entry_id)) {
6528		- wp_send_json_error('Invalid entry ID');
6529		- exit;
6530		- }
6531		-
6532		- // Get knowledge manager instance to validate role restriction
6533		- $knowledge_manager = MxChat_Knowledge_Manager::get_instance();
6534		- $valid_roles = array_keys($knowledge_manager->mxchat_get_role_options());
6535		- if (!in_array($role_restriction, $valid_roles)) {
6536		- wp_send_json_error('Invalid role restriction');
6537		- exit;
6538		- }
6539		-
6540		- global $wpdb;
6541		-
6542		- if ($data_source === 'pinecone') {
6543		- // Handle Pinecone role restriction (stored separately in WordPress table)
6544		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6545	3269
6546		- // Use REPLACE to insert or update the role restriction
6547		- $result = $wpdb->replace(
6548		- $roles_table,
6549		- array(
6550		- 'vector_id' => $entry_id,
6551		- 'role_restriction' => $role_restriction,
6552		- 'updated_at' => current_time('mysql')
6553		- ),
6554		- array('%s', '%s', '%s')
6555		- );
6556		-
6557		- // No cache clearing needed since we removed caching
6558		-
6559		- } else {
6560		- // Handle WordPress database role restriction (existing functionality)
6561		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6562		-
6563		- $result = $wpdb->update(
	3270	+ $wpdb->delete(
6564	3271	$table_name,
6565		- array('role_restriction' => $role_restriction),
6566		- array('id' => absint($entry_id)),
6567		- array('%s'),
6568		- array('%d')
6569		- );
6570		- }
6571		-
6572		- if ($result === false) {
6573		- wp_send_json_error('Database update failed: ' . $wpdb->last_error);
6574		- exit;
6575		- }
6576		-
6577		- wp_send_json_success(array(
6578		- 'message' => 'Role restriction updated successfully',
6579		- 'role_restriction' => $role_restriction,
6580		- 'data_source' => $data_source,
6581		- 'entry_id' => $entry_id
6582		- ));
6583		- exit;
6584		-}
6585		-
6586		-// ========================================
6587		-// ROLE-BASED CONTENT RESTRICTIONS - NEW FUNCTIONS
6588		-// Add these to your MxChat_Knowledge_Manager class
6589		-// ========================================
6590		-
6591		-/**
6592		- * Initialize role-based content hooks
6593		- * Add this call to your __construct() or mxchat_init_hooks() method
6594		- */
6595		-private function mxchat_init_role_hooks() {
6596		- // AJAX handlers for tag-role mappings
6597		- add_action('wp_ajax_mxchat_add_tag_role_mapping', array($this, 'ajax_add_tag_role_mapping'));
6598		- add_action('wp_ajax_mxchat_delete_tag_role_mapping', array($this, 'ajax_delete_tag_role_mapping'));
6599		- add_action('wp_ajax_mxchat_get_tag_role_mappings', array($this, 'ajax_get_tag_role_mappings'));
6600		- add_action('wp_ajax_mxchat_bulk_update_tag_roles', array($this, 'ajax_bulk_update_tag_roles'));
6601		-
6602		- // Hook to automatically update role restrictions when tags are added/removed
6603		- add_action('set_object_terms', array($this, 'handle_tag_change'), 10, 6);
6604		-
6605		- // Hook to apply role restrictions on auto-sync
6606		- add_action('mxchat_content_stored', array($this, 'apply_role_restriction_after_storage'), 10, 2);
6607		-}
6608		-
6609		-/**
6610		- * Add tag-role mapping via AJAX
6611		- */
6612		-public function ajax_add_tag_role_mapping() {
6613		- // Verify nonce and permissions
6614		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6615		-
6616		- if (!current_user_can('manage_options')) {
6617		- wp_send_json_error('Unauthorized access');
6618		- exit;
6619		- }
6620		-
6621		- $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
6622		- $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
6623		-
6624		- if (empty($tag_slug)) {
6625		- wp_send_json_error('Tag slug is required');
6626		- exit;
6627		- }
6628		-
6629		- // Validate role restriction
6630		- $valid_roles = array_keys($this->mxchat_get_role_options());
6631		- if (!in_array($role_restriction, $valid_roles)) {
6632		- wp_send_json_error('Invalid role restriction');
6633		- exit;
6634		- }
6635		-
6636		- // Check if tag exists in WordPress
6637		- $term = get_term_by('slug', $tag_slug, 'post_tag');
6638		- if (!$term) {
6639		- wp_send_json_error('Tag does not exist in WordPress');
6640		- exit;
6641		- }
6642		-
6643		- // Get existing mappings
6644		- $mappings = get_option('mxchat_tag_role_mappings', array());
6645		-
6646		- // Check if mapping already exists
6647		- if (isset($mappings[$tag_slug])) {
6648		- wp_send_json_error('Mapping for this tag already exists');
6649		- exit;
6650		- }
6651		-
6652		- // Add new mapping
6653		- $mappings[$tag_slug] = $role_restriction;
6654		- update_option('mxchat_tag_role_mappings', $mappings);
6655		-
6656		- wp_send_json_success(array(
6657		- 'message' => 'Tag-role mapping added successfully',
6658		- 'tag_slug' => $tag_slug,
6659		- 'role_restriction' => $role_restriction
6660		- ));
6661		- exit;
6662		-}
6663		-
6664		-/**
6665		- * Delete tag-role mapping via AJAX
6666		- */
6667		-public function ajax_delete_tag_role_mapping() {
6668		- // Verify nonce and permissions
6669		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6670		-
6671		- if (!current_user_can('manage_options')) {
6672		- wp_send_json_error('Unauthorized access');
6673		- exit;
6674		- }
6675		-
6676		- $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
6677		-
6678		- if (empty($tag_slug)) {
6679		- wp_send_json_error('Tag slug is required');
6680		- exit;
6681		- }
6682		-
6683		- // Get existing mappings
6684		- $mappings = get_option('mxchat_tag_role_mappings', array());
6685		-
6686		- // Check if mapping exists
6687		- if (!isset($mappings[$tag_slug])) {
6688		- wp_send_json_error('Mapping does not exist');
6689		- exit;
6690		- }
6691		-
6692		- // Remove mapping
6693		- unset($mappings[$tag_slug]);
6694		- update_option('mxchat_tag_role_mappings', $mappings);
6695		-
6696		- wp_send_json_success(array(
6697		- 'message' => 'Tag-role mapping deleted successfully',
6698		- 'tag_slug' => $tag_slug
6699		- ));
6700		- exit;
6701		-}
6702		-
6703		-/**
6704		- * Get all tag-role mappings via AJAX
6705		- */
6706		-public function ajax_get_tag_role_mappings() {
6707		- // Verify nonce and permissions
6708		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6709		-
6710		- if (!current_user_can('manage_options')) {
6711		- wp_send_json_error('Unauthorized access');
6712		- exit;
6713		- }
6714		-
6715		- // Get mappings
6716		- $mappings = get_option('mxchat_tag_role_mappings', array());
6717		- $role_options = $this->mxchat_get_role_options();
6718		-
6719		- $formatted_mappings = array();
6720		-
6721		- foreach ($mappings as $tag_slug => $role_restriction) {
6722		- // Get tag object
6723		- $term = get_term_by('slug', $tag_slug, 'post_tag');
6724		-
6725		- // Count posts with this tag
6726		- $post_count = 0;
6727		- if ($term) {
6728		- $post_count = $term->count;
6729		- }
6730		-
6731		- $formatted_mappings[] = array(
6732		- 'tag_slug' => $tag_slug,
6733		- 'role_restriction' => $role_restriction,
6734		- 'role_label' => $role_options[$role_restriction] ?? $role_restriction,
6735		- 'post_count' => $post_count
6736		- );
6737		- }
6738		-
6739		- wp_send_json_success(array(
6740		- 'mappings' => $formatted_mappings
6741		- ));
6742		- exit;
6743		-}
6744		-
6745		-/**
6746		- * Bulk update role restrictions for all existing content with mapped tags
6747		- */
6748		-public function ajax_bulk_update_tag_roles() {
6749		- // Verify nonce and permissions
6750		- check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
6751		-
6752		- if (!current_user_can('manage_options')) {
6753		- wp_send_json_error('Unauthorized access');
6754		- exit;
6755		- }
6756		-
6757		- // Get mappings
6758		- $mappings = get_option('mxchat_tag_role_mappings', array());
6759		-
6760		- if (empty($mappings)) {
6761		- wp_send_json_error('No tag-role mappings found');
6762		- exit;
6763		- }
6764		-
6765		- global $wpdb;
6766		-
6767		- // Check if using Pinecone
6768		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6769		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6770		-
6771		- $updated_count = 0;
6772		- $details = array();
6773		-
6774		- foreach ($mappings as $tag_slug => $role_restriction) {
6775		- // Get all posts with this tag
6776		- $posts = get_posts(array(
6777		- 'tag' => $tag_slug,
6778		- 'post_type' => 'any',
6779		- 'posts_per_page' => -1,
6780		- 'fields' => 'ids',
6781		- 'post_status' => 'publish'
6782		- ));
6783		-
6784		- if (empty($posts)) {
6785		- continue;
6786		- }
6787		-
6788		- $tag_updated = 0;
6789		-
6790		- foreach ($posts as $post_id) {
6791		- $source_url = get_permalink($post_id);
6792		- if (!$source_url) {
6793		- continue;
6794		- }
6795		-
6796		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
6797		- // Update Pinecone role restriction
6798		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6799		- $vector_id = md5($source_url);
6800		-
6801		- $result = $wpdb->replace(
6802		- $roles_table,
6803		- array(
6804		- 'vector_id' => $vector_id,
6805		- 'role_restriction' => $role_restriction,
6806		- 'updated_at' => current_time('mysql')
6807		- ),
6808		- array('%s', '%s', '%s')
6809		- );
6810		- } else {
6811		- // Update WordPress DB
6812		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6813		-
6814		- $result = $wpdb->update(
6815		- $table_name,
6816		- array('role_restriction' => $role_restriction),
6817		- array('source_url' => $source_url),
6818		- array('%s'),
6819		- array('%s')
6820		- );
6821		- }
6822		-
6823		- if ($result !== false) {
6824		- $tag_updated++;
6825		- $updated_count++;
6826		- }
6827		- }
6828		-
6829		- if ($tag_updated > 0) {
6830		- $details[] = sprintf(
6831		- 'Tag "%s" (%s): %d posts updated',
6832		- $tag_slug,
6833		- $role_restriction,
6834		- $tag_updated
6835		- );
6836		- }
6837		- }
6838		-
6839		- wp_send_json_success(array(
6840		- 'message' => 'Bulk update completed',
6841		- 'updated_count' => $updated_count,
6842		- 'tags_processed' => count($mappings),
6843		- 'details' => $details
6844		- ));
6845		- exit;
6846		-}
6847		-
6848		-/**
6849		- * Handle tag changes on posts (when tags are added or removed)
6850		- */
6851		-public function handle_tag_change($object_id, $terms, $tt_ids, $taxonomy, $append, $old_tt_ids) {
6852		- // Only process post tags
6853		- if ($taxonomy !== 'post_tag') {
6854		- return;
6855		- }
6856		-
6857		- // Get tag-role mappings
6858		- $mappings = get_option('mxchat_tag_role_mappings', array());
6859		-
6860		- if (empty($mappings)) {
6861		- return;
6862		- }
6863		-
6864		- // Get the post's URL
6865		- $source_url = get_permalink($object_id);
6866		- if (!$source_url) {
6867		- return;
6868		- }
6869		-
6870		- // Determine the highest role restriction based on tags
6871		- $highest_role = 'public';
6872		- $role_hierarchy = array(
6873		- 'public' => 0,
6874		- 'logged_in' => 1,
6875		- 'subscriber' => 2,
6876		- 'contributor' => 3,
6877		- 'author' => 4,
6878		- 'editor' => 5,
6879		- 'administrator' => 6
6880		- );
6881		-
6882		- // Get all current tags for the post
6883		- $current_tags = wp_get_post_tags($object_id, array('fields' => 'slugs'));
6884		-
6885		- // Find the highest role restriction among the tags
6886		- foreach ($current_tags as $tag_slug) {
6887		- if (isset($mappings[$tag_slug])) {
6888		- $role = $mappings[$tag_slug];
6889		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
6890		- $highest_role = $role;
6891		- }
6892		- }
6893		- }
6894		-
6895		- // Update the role restriction in the database
6896		- global $wpdb;
6897		-
6898		- // Check if using Pinecone
6899		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6900		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6901		-
6902		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
6903		- // Update Pinecone role restriction
6904		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6905		- $vector_id = md5($source_url);
6906		-
6907		- $wpdb->replace(
6908		- $roles_table,
6909		- array(
6910		- 'vector_id' => $vector_id,
6911		- 'role_restriction' => $highest_role,
6912		- 'updated_at' => current_time('mysql')
6913		- ),
6914		- array('%s', '%s', '%s')
6915		- );
6916		- } else {
6917		- // Update WordPress DB
6918		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6919		-
6920		- $wpdb->update(
6921		- $table_name,
6922		- array('role_restriction' => $highest_role),
6923	3272	array('source_url' => $source_url),
6924		- array('%s'),
6925	3273	array('%s')
6926	3274	);
6927	3275	}
6928	3276	}
6929		-
6930		-/**
6931		- * Apply role restriction after content is stored (for auto-sync)
6932		- */
6933		-public function apply_role_restriction_after_storage($post_id, $source_url) {
6934		- // Get tag-role mappings
6935		- $mappings = get_option('mxchat_tag_role_mappings', array());
6936	3277
6937		- if (empty($mappings)) {
6938		- return;
6939		- }
6940		-
6941		- // Get all tags for the post
6942		- $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
6943		-
6944		- if (empty($post_tags)) {
6945		- return;
6946		- }
6947		-
6948		- // Determine the highest role restriction based on tags
6949		- $highest_role = 'public';
6950		- $role_hierarchy = array(
6951		- 'public' => 0,
6952		- 'logged_in' => 1,
6953		- 'subscriber' => 2,
6954		- 'contributor' => 3,
6955		- 'author' => 4,
6956		- 'editor' => 5,
6957		- 'administrator' => 6
6958		- );
6959		-
6960		- foreach ($post_tags as $tag_slug) {
6961		- if (isset($mappings[$tag_slug])) {
6962		- $role = $mappings[$tag_slug];
6963		- if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
6964		- $highest_role = $role;
6965		- }
6966		- }
6967		- }
6968		-
6969		- // If no restricted tags found, return (leave as public)
6970		- if ($highest_role === 'public') {
6971		- return;
6972		- }
6973		-
6974		- // Update the role restriction
6975		- global $wpdb;
6976		-
6977		- // Check if using Pinecone
6978		- $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6979		- $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6980		-
6981		- if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
6982		- // Update Pinecone role restriction
6983		- $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
6984		- $vector_id = md5($source_url);
6985		-
6986		- $wpdb->replace(
6987		- $roles_table,
6988		- array(
6989		- 'vector_id' => $vector_id,
6990		- 'role_restriction' => $highest_role,
6991		- 'updated_at' => current_time('mysql')
6992		- ),
6993		- array('%s', '%s', '%s')
6994		- );
6995		- } else {
6996		- // Update WordPress DB
6997		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6998		-
6999		- $wpdb->update(
7000		- $table_name,
7001		- array('role_restriction' => $highest_role),
7002		- array('source_url' => $source_url),
7003		- array('%s'),
7004		- array('%s')
7005		- );
7006		- }
7007		-}
7008		-
7009		-
7010	3278	// ========================================
7011	3279	// HELPER METHODS
7012	3280	// ========================================
7013	3281
		@@ -7074,1074 +3342,8 @@
7074	3342	*/
7075	3343	private function mxchat_get_pinecone_manager() {
7076	3344	return MxChat_Pinecone_Manager::get_instance();
7077	3345	}
7078		-
7079		-
7080		- // ========================================
7081		-// DATABASE QUEUE TABLE MANAGEMENT
7082		-// ========================================
7083		-
7084		-/**
7085		- * Create queue table on plugin activation
7086		- * Call this from your plugin activation hook
7087		- */
7088		-public function mxchat_create_queue_table() {
7089		- global $wpdb;
7090		-
7091		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7092		- $charset_collate = $wpdb->get_charset_collate();
7093		-
7094		- $sql = "CREATE TABLE IF NOT EXISTS $table_name (
7095		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
7096		- queue_id varchar(64) NOT NULL,
7097		- item_type varchar(20) NOT NULL,
7098		- item_data longtext NOT NULL,
7099		- status varchar(20) NOT NULL DEFAULT 'pending',
7100		- bot_id varchar(50) NOT NULL DEFAULT 'default',
7101		- priority int(11) NOT NULL DEFAULT 0,
7102		- attempts int(11) NOT NULL DEFAULT 0,
7103		- max_attempts int(11) NOT NULL DEFAULT 3,
7104		- error_message text DEFAULT NULL,
7105		- created_at datetime NOT NULL,
7106		- started_at datetime DEFAULT NULL,
7107		- completed_at datetime DEFAULT NULL,
7108		- PRIMARY KEY (id),
7109		- KEY queue_id (queue_id),
7110		- KEY status (status),
7111		- KEY item_type (item_type),
7112		- KEY priority (priority)
7113		- ) $charset_collate;";
7114		-
7115		- require_once(ABSPATH . 'wp-admin/includes/upgrade.php');
7116		- dbDelta($sql);
7117		-
7118		- // Also create a meta table for queue metadata
7119		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7120		-
7121		- $meta_sql = "CREATE TABLE IF NOT EXISTS $meta_table (
7122		- id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
7123		- queue_id varchar(64) NOT NULL,
7124		- meta_key varchar(255) NOT NULL,
7125		- meta_value longtext,
7126		- PRIMARY KEY (id),
7127		- KEY queue_id (queue_id),
7128		- KEY meta_key (meta_key)
7129		- ) $charset_collate;";
7130		-
7131		- dbDelta($meta_sql);
7132		-}
7133		-
7134		-/**
7135		- * Add items to the processing queue
7136		- *
7137		- * @param string $queue_id Unique identifier for this queue batch
7138		- * @param string $item_type Type of item (url, pdf_page)
7139		- * @param array $items Array of items to queue
7140		- * @param string $bot_id Bot ID for processing
7141		- * @return int Number of items queued
7142		- */
7143		-private function mxchat_add_to_queue($queue_id, $item_type, $items, $bot_id = 'default') {
7144		- global $wpdb;
7145		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7146		-
7147		- $queued_count = 0;
7148		- $priority = 0;
7149		-
7150		- foreach ($items as $item) {
7151		- $result = $wpdb->insert(
7152		- $table_name,
7153		- array(
7154		- 'queue_id' => $queue_id,
7155		- 'item_type' => $item_type,
7156		- 'item_data' => wp_json_encode($item),
7157		- 'status' => 'pending',
7158		- 'bot_id' => $bot_id,
7159		- 'priority' => $priority,
7160		- 'attempts' => 0,
7161		- 'max_attempts' => 3,
7162		- 'created_at' => current_time('mysql')
7163		- ),
7164		- array('%s', '%s', '%s', '%s', '%s', '%d', '%d', '%d', '%s')
7165		- );
7166		-
7167		- if ($result) {
7168		- $queued_count++;
7169		- }
7170		-
7171		- $priority++; // Process in order
7172		- }
7173		-
7174		- return $queued_count;
7175		-}
7176		-
7177		-/**
7178		- * Store queue metadata (total counts, source URL, etc.)
7179		- */
7180		-private function mxchat_set_queue_meta($queue_id, $meta_key, $meta_value) {
7181		- global $wpdb;
7182		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7183		-
7184		- // Check if meta exists
7185		- $existing = $wpdb->get_var($wpdb->prepare(
7186		- "SELECT id FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
7187		- $queue_id,
7188		- $meta_key
7189		- ));
7190		-
7191		- if ($existing) {
7192		- // Update
7193		- $wpdb->update(
7194		- $meta_table,
7195		- array('meta_value' => maybe_serialize($meta_value)),
7196		- array('queue_id' => $queue_id, 'meta_key' => $meta_key),
7197		- array('%s'),
7198		- array('%s', '%s')
7199		- );
7200		- } else {
7201		- // Insert
7202		- $wpdb->insert(
7203		- $meta_table,
7204		- array(
7205		- 'queue_id' => $queue_id,
7206		- 'meta_key' => $meta_key,
7207		- 'meta_value' => maybe_serialize($meta_value)
7208		- ),
7209		- array('%s', '%s', '%s')
7210		- );
7211		- }
7212		-}
7213		-
7214		-/**
7215		- * Get queue metadata
7216		- */
7217		-private function mxchat_get_queue_meta($queue_id, $meta_key) {
7218		- global $wpdb;
7219		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7220		-
7221		- $value = $wpdb->get_var($wpdb->prepare(
7222		- "SELECT meta_value FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
7223		- $queue_id,
7224		- $meta_key
7225		- ));
7226		-
7227		- return maybe_unserialize($value);
7228		-}
7229		-
7230		-// ========================================
7231		-// AJAX QUEUE PROCESSING HANDLERS
7232		-// ========================================
7233		-
7234		-/**
7235		- * AJAX: Get next item from queue to process
7236		- */
7237		-public function ajax_mxchat_get_next_queue_item() {
7238		- // Verify nonce and permissions
7239		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7240		-
7241		- if (!current_user_can('manage_options')) {
7242		- wp_send_json_error('Unauthorized access');
7243		- }
7244		-
7245		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7246		-
7247		- if (empty($queue_id)) {
7248		- wp_send_json_error('Missing queue ID');
7249		- }
7250		-
7251		- global $wpdb;
7252		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7253		-
7254		- // Get next pending item with retry logic for failed items
7255		- $next_item = $wpdb->get_row($wpdb->prepare(
7256		- "SELECT * FROM $table_name
7257		- WHERE queue_id = %s
7258		- AND status IN ('pending', 'failed')
7259		- AND attempts < max_attempts
7260		- ORDER BY priority ASC, id ASC
7261		- LIMIT 1",
7262		- $queue_id
7263		- ));
7264		-
7265		- if (!$next_item) {
7266		- // No more items - queue complete
7267		- wp_send_json_success(array(
7268		- 'complete' => true,
7269		- 'message' => 'Queue processing complete'
7270		- ));
7271		- }
7272		-
7273		- // Mark item as processing
7274		- $wpdb->update(
7275		- $table_name,
7276		- array(
7277		- 'status' => 'processing',
7278		- 'started_at' => current_time('mysql'),
7279		- 'attempts' => $next_item->attempts + 1
7280		- ),
7281		- array('id' => $next_item->id),
7282		- array('%s', '%s', '%d'),
7283		- array('%d')
7284		- );
7285		-
7286		- wp_send_json_success(array(
7287		- 'complete' => false,
7288		- 'item' => array(
7289		- 'id' => $next_item->id,
7290		- 'type' => $next_item->item_type,
7291		- 'data' => json_decode($next_item->item_data, true),
7292		- 'bot_id' => $next_item->bot_id,
7293		- 'attempt' => $next_item->attempts + 1
7294		- )
7295		- ));
7296		-}
7297		-
7298		-/**
7299		- * AJAX: Process a single queue item
7300		- */
7301		-public function ajax_mxchat_process_queue_item() {
7302		- // Verify nonce and permissions
7303		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7304		-
7305		- if (!current_user_can('manage_options')) {
7306		- wp_send_json_error('Unauthorized access');
7307		- }
7308		-
7309		- $item_id = isset($_POST['item_id']) ? absint($_POST['item_id']) : 0;
7310		- $item_type = isset($_POST['item_type']) ? sanitize_text_field($_POST['item_type']) : '';
7311		- $item_data = isset($_POST['item_data']) ? $_POST['item_data'] : array();
7312		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
7313		-
7314		- if (empty($item_id) || empty($item_type)) {
7315		- wp_send_json_error('Missing item data');
7316		- }
7317		-
7318		- global $wpdb;
7319		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7320		-
7321		- // Process based on item type
7322		- try {
7323		- set_time_limit(60); // Give processing 60 seconds
7324		-
7325		- $result = false;
7326		- $error_message = '';
7327		-
7328		- // Read item directly from DB to get queue_id and preserve special chars in item_data
7329		- // (POST round-trip through JS mangles characters like apostrophes in URLs)
7330		- $db_item = $wpdb->get_row($wpdb->prepare(
7331		- "SELECT queue_id, item_data FROM $table_name WHERE id = %d",
7332		- $item_id
7333		- ));
7334		- $item_queue_id = $db_item ? $db_item->queue_id : '';
7335		- if ($db_item && !empty($db_item->item_data)) {
7336		- $db_data = json_decode($db_item->item_data, true);
7337		- if (is_array($db_data)) {
7338		- $item_data = $db_data;
7339		- }
7340		- }
7341		-
7342		- switch ($item_type) {
7343		- case 'url':
7344		- $result = $this->mxchat_process_queue_url($item_data, $bot_id, $item_queue_id);
7345		- break;
7346		-
7347		- case 'pdf_page':
7348		- $result = $this->mxchat_process_queue_pdf_page($item_data, $bot_id);
7349		- break;
7350		-
7351		- default:
7352		- throw new Exception('Unknown item type: ' . $item_type);
7353		- }
7354		-
7355		- if (is_wp_error($result)) {
7356		- $error_code = $result->get_error_code();
7357		- // Content errors (empty page, sanitization) are permanent — retrying won't help
7358		- $permanent_codes = array('empty_page', 'empty_after_sanitization', 'no_api_key', 'page_not_found');
7359		- if (in_array($error_code, $permanent_codes)) {
7360		- // Mark as permanently failed — set attempts = max_attempts so it won't be retried
7361		- $current_item = $wpdb->get_row($wpdb->prepare(
7362		- "SELECT max_attempts FROM $table_name WHERE id = %d", $item_id
7363		- ));
7364		- $wpdb->update(
7365		- $table_name,
7366		- array(
7367		- 'status' => 'failed',
7368		- 'error_message' => $result->get_error_message(),
7369		- 'attempts' => $current_item ? $current_item->max_attempts : 3
7370		- ),
7371		- array('id' => $item_id),
7372		- array('%s', '%s', '%d'),
7373		- array('%d')
7374		- );
7375		- wp_send_json_error(array(
7376		- 'message' => $result->get_error_message(),
7377		- 'permanent_failure' => true,
7378		- 'item_id' => $item_id
7379		- ));
7380		- return;
7381		- }
7382		- throw new Exception($result->get_error_message());
7383		- }
7384		-
7385		- if ($result === false) {
7386		- throw new Exception('Processing returned false - item may be empty or invalid');
7387		- }
7388		-
7389		- // Mark as completed
7390		- $wpdb->update(
7391		- $table_name,
7392		- array(
7393		- 'status' => 'completed',
7394		- 'completed_at' => current_time('mysql'),
7395		- 'error_message' => null
7396		- ),
7397		- array('id' => $item_id),
7398		- array('%s', '%s', '%s'),
7399		- array('%d')
7400		- );
7401		-
7402		- wp_send_json_success(array(
7403		- 'processed' => true,
7404		- 'item_id' => $item_id,
7405		- 'message' => 'Item processed successfully'
7406		- ));
7407		-
7408		- } catch (Exception $e) {
7409		- $error_message = $e->getMessage();
7410		-
7411		- // Get current attempt count
7412		- $item = $wpdb->get_row($wpdb->prepare(
7413		- "SELECT attempts, max_attempts FROM $table_name WHERE id = %d",
7414		- $item_id
7415		- ));
7416		-
7417		- // Check if we've exhausted retries
7418		- if ($item && $item->attempts >= $item->max_attempts) {
7419		- // Permanently failed
7420		- $wpdb->update(
7421		- $table_name,
7422		- array(
7423		- 'status' => 'failed',
7424		- 'error_message' => $error_message
7425		- ),
7426		- array('id' => $item_id),
7427		- array('%s', '%s'),
7428		- array('%d')
7429		- );
7430		-
7431		- wp_send_json_error(array(
7432		- 'message' => 'Item failed after maximum attempts: ' . $error_message,
7433		- 'permanent_failure' => true,
7434		- 'item_id' => $item_id
7435		- ));
7436		- } else {
7437		- // Mark for retry
7438		- $wpdb->update(
7439		- $table_name,
7440		- array(
7441		- 'status' => 'failed',
7442		- 'error_message' => $error_message
7443		- ),
7444		- array('id' => $item_id),
7445		- array('%s', '%s'),
7446		- array('%d')
7447		- );
7448		-
7449		- wp_send_json_error(array(
7450		- 'message' => 'Item processing failed, will retry: ' . $error_message,
7451		- 'can_retry' => true,
7452		- 'item_id' => $item_id,
7453		- 'attempts' => $item ? $item->attempts : 0
7454		- ));
7455		- }
7456		- }
7457		-}
7458		-
7459		-/**
7460		- * Process a URL from the queue
7461		- */
7462		-private function mxchat_process_queue_url($item_data, $bot_id = 'default', $queue_id = '') {
7463		- $url = isset($item_data['url']) ? $item_data['url'] : '';
7464		-
7465		- if (empty($url)) {
7466		- return new WP_Error('invalid_url', 'URL is empty');
7467		- }
7468		-
7469		- // Get bot-specific API key early (needed for both paths)
7470		- $bot_options = $this->get_bot_options($bot_id);
7471		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
7472		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
7473		-
7474		- if (strpos($selected_model, 'voyage') === 0) {
7475		- $api_key = $options['voyage_api_key'] ?? '';
7476		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
7477		- $api_key = $options['gemini_api_key'] ?? '';
7478		- } else {
7479		- $api_key = $options['api_key'] ?? '';
7480		- }
7481		-
7482		- if (empty($api_key)) {
7483		- return new WP_Error('no_api_key', 'API key not configured for bot: ' . $bot_id);
7484		- }
7485		-
7486		- // Check if this is a WooCommerce product URL and WooCommerce is active
7487		- $is_product_url = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
7488		- $content_type = $is_product_url ? 'product' : 'url';
7489		-
7490		- // Try to get WooCommerce product data if it's a product URL
7491		- if ($is_product_url && class_exists('WooCommerce')) {
7492		- $product_content = $this->mxchat_extract_woocommerce_product_content($url);
7493		-
7494		- if (!empty($product_content)) {
7495		- // Successfully extracted WooCommerce product data with pricing
7496		- $result = MxChat_Utils::submit_content_to_db(
7497		- $product_content,
7498		- $url,
7499		- $api_key,
7500		- null,
7501		- $bot_id,
7502		- 'product'
7503		- );
7504		- return $result;
7505		- }
7506		- // If WooCommerce extraction failed, fall through to HTML extraction
7507		- }
7508		-
7509		- // Fetch URL content (fallback for non-products or when WooCommerce extraction fails)
7510		- $is_likely_pdf = (strtolower(pathinfo(parse_url($url, PHP_URL_PATH) ?: '', PATHINFO_EXTENSION)) === 'pdf');
7511		- $response = wp_remote_get($url, array(
7512		- 'timeout' => $is_likely_pdf ? 120 : 30,
7513		- 'redirection' => 5,
7514		- 'user-agent' => 'MxChat/1.0'
7515		- ));
7516		-
7517		- if (is_wp_error($response)) {
7518		- return $response;
7519		- }
7520		-
7521		- $response_code = wp_remote_retrieve_response_code($response);
7522		- if ($response_code !== 200) {
7523		- return new WP_Error('http_error', 'HTTP ' . $response_code . ' error for: ' . $url);
7524		- }
7525		-
7526		- // Check if URL is a PDF — expand into per-page queue items using the standard PDF pipeline
7527		- if ($this->mxchat_is_pdf_url($url, $response)) {
7528		- return $this->mxchat_expand_pdf_to_queue($url, $response, $bot_id, $queue_id);
7529		- }
7530		-
7531		- $html = wp_remote_retrieve_body($response);
7532		-
7533		- if (empty($html)) {
7534		- return new WP_Error('empty_response', 'Empty response body');
7535		- }
7536		-
7537		- // Extract and sanitize content
7538		- $content = $this->mxchat_extract_main_content($html);
7539		- $sanitized = $this->mxchat_sanitize_content_for_api($content);
7540		-
7541		- if (empty($sanitized)) {
7542		- // Not an error - just no content found (maybe a redirect or empty page)
7543		- return false;
7544		- }
7545		-
7546		- // Submit to database with content_type
7547		- $result = MxChat_Utils::submit_content_to_db(
7548		- $sanitized,
7549		- $url,
7550		- $api_key,
7551		- null,
7552		- $bot_id,
7553		- $content_type
7554		- );
7555		-
7556		- return $result;
7557		-}
7558		-
7559		-/**
7560		- * Expand a PDF URL into per-page queue items using the standard PDF pipeline.
7561		- * Called when a sitemap URL turns out to be a PDF — downloads, parses page count,
7562		- * and adds pdf_page items to the same queue so they process with full progress tracking.
7563		- */
7564		-private function mxchat_expand_pdf_to_queue($pdf_url, $response, $bot_id = 'default', $queue_id = '') {
7565		- set_time_limit(120); // PDFs need extra time for download + parsing
7566		-
7567		- $upload_dir = wp_upload_dir();
7568		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
7569		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
7570		-
7571		- $response_body = wp_remote_retrieve_body($response);
7572		- if (empty($response_body)) {
7573		- return new WP_Error('empty_pdf', 'Empty PDF response for: ' . $pdf_url);
7574		- }
7575		-
7576		- if (!wp_mkdir_p(dirname($pdf_path))) {
7577		- return new WP_Error('dir_error', 'Failed to create upload directory');
7578		- }
7579		-
7580		- file_put_contents($pdf_path, $response_body);
7581		-
7582		- if (!file_exists($pdf_path)) {
7583		- return new WP_Error('save_error', 'Failed to save PDF file');
7584		- }
7585		-
7586		- try {
7587		- $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
7588		-
7589		- if ($total_pages === false || $total_pages < 1) {
7590		- wp_delete_file($pdf_path);
7591		- return new WP_Error('no_pages', 'PDF has no pages: ' . $pdf_url);
7592		- }
7593		-
7594		- // Build per-page items identical to mxchat_handle_pdf_for_knowledge_base
7595		- $pages = array();
7596		- for ($i = 1; $i <= $total_pages; $i++) {
7597		- $pages[] = array(
7598		- 'pdf_path' => $pdf_path,
7599		- 'pdf_url' => $pdf_url,
7600		- 'page_number' => $i,
7601		- 'total_pages' => $total_pages
7602		- );
7603		- }
7604		-
7605		- // Add pdf_page items to the SAME queue so the JS picks them up automatically
7606		- if (!empty($queue_id)) {
7607		- $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
7608		- } else {
7609		- // Fallback: create a new PDF queue (shouldn't happen in sitemap flow)
7610		- $new_queue_id = 'pdf_' . md5($pdf_url . time());
7611		- $queued_count = $this->mxchat_add_to_queue($new_queue_id, 'pdf_page', $pages, $bot_id);
7612		- $this->mxchat_set_queue_meta($new_queue_id, 'source_url', $pdf_url);
7613		- $this->mxchat_set_queue_meta($new_queue_id, 'queue_type', 'pdf');
7614		- $this->mxchat_set_queue_meta($new_queue_id, 'total_items', $total_pages);
7615		- $this->mxchat_set_queue_meta($new_queue_id, 'bot_id', $bot_id);
7616		- $this->mxchat_set_queue_meta($new_queue_id, 'pdf_path', $pdf_path);
7617		- $this->mxchat_set_queue_meta($new_queue_id, 'created_at', current_time('mysql'));
7618		- }
7619		-
7620		- if ($queued_count === 0) {
7621		- wp_delete_file($pdf_path);
7622		- return new WP_Error('queue_error', 'Failed to add PDF pages to queue');
7623		- }
7624		-
7625		- // Return true so the original URL item is marked complete
7626		- // The new pdf_page items will be processed in subsequent batches
7627		- return true;
7628		-
7629		- } catch (Exception $e) {
7630		- if (file_exists($pdf_path)) {
7631		- wp_delete_file($pdf_path);
7632		- }
7633		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
7634		- }
7635		-}
7636		-
7637		-/**
7638		- * Legacy: Process a PDF URL inline during sitemap queue processing.
7639		- * @deprecated Use mxchat_expand_pdf_to_queue instead — kept for reference only.
7640		- */
7641		-private function mxchat_process_pdf_url_inline($pdf_url, $response, $api_key, $bot_id = 'default') {
7642		- set_time_limit(120); // PDFs need more time — downloading + parsing all pages
7643		-
7644		- $upload_dir = wp_upload_dir();
7645		- $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
7646		- $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
7647		-
7648		- $response_body = wp_remote_retrieve_body($response);
7649		- if (empty($response_body)) {
7650		- return new WP_Error('empty_pdf', 'Empty PDF response');
7651		- }
7652		-
7653		- if (!wp_mkdir_p(dirname($pdf_path))) {
7654		- return new WP_Error('dir_error', 'Failed to create upload directory');
7655		- }
7656		-
7657		- file_put_contents($pdf_path, $response_body);
7658		-
7659		- if (!file_exists($pdf_path)) {
7660		- return new WP_Error('save_error', 'Failed to save PDF file');
7661		- }
7662		-
7663		- try {
7664		- mxchat_load_pdf_parser();
7665		- $parser = new \Smalot\PdfParser\Parser();
7666		- $pdf = $parser->parseFile($pdf_path);
7667		- $pages = $pdf->getPages();
7668		- $total_pages = count($pages);
7669		-
7670		- if ($total_pages < 1) {
7671		- wp_delete_file($pdf_path);
7672		- return new WP_Error('no_pages', 'PDF has no pages');
7673		- }
7674		-
7675		- $processed = 0;
7676		- $skipped_pages = array();
7677		-
7678		- for ($i = 0; $i < $total_pages; $i++) {
7679		- $page_num = $i + 1;
7680		- $text = $pages[$i]->getText();
7681		- if (empty($text)) {
7682		- $skipped_pages[] = 'Page ' . $page_num . ': No text could be extracted — page may contain only images, links, or non-standard encoding';
7683		- continue;
7684		- }
7685		-
7686		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
7687		- if (empty($sanitized)) {
7688		- $skipped_pages[] = 'Page ' . $page_num . ': Text was extracted but contained only special characters, control codes, or unsupported content';
7689		- continue;
7690		- }
7691		-
7692		- $metadata = array(
7693		- 'document_type' => 'pdf',
7694		- 'total_pages' => $total_pages,
7695		- 'current_page' => $page_num,
7696		- 'source_url' => $pdf_url,
7697		- );
7698		-
7699		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
7700		- $page_url = esc_url($pdf_url . '#page=' . $page_num);
7701		-
7702		- MxChat_Utils::submit_content_to_db(
7703		- $content_with_metadata,
7704		- $page_url,
7705		- $api_key,
7706		- null,
7707		- $bot_id,
7708		- 'pdf'
7709		- );
7710		-
7711		- $processed++;
7712		- }
7713		-
7714		- // Clean up the temp PDF file
7715		- wp_delete_file($pdf_path);
7716		-
7717		- if (!empty($skipped_pages)) {
7718		- error_log('MxChat PDF: Skipped ' . count($skipped_pages) . ' of ' . $total_pages . ' pages: ' . implode('; ', $skipped_pages));
7719		- }
7720		-
7721		- return $processed > 0 ? true : false;
7722		-
7723		- } catch (Exception $e) {
7724		- if (file_exists($pdf_path)) {
7725		- wp_delete_file($pdf_path);
7726		- }
7727		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
7728		- }
7729		-}
7730		-
7731		-/**
7732		- * Extract WooCommerce product content including pricing
7733		- *
7734		- * @param string $url The product URL
7735		- * @return string|false Product content with pricing, or false if not found
7736		- */
7737		-private function mxchat_extract_woocommerce_product_content($url) {
7738		- // Try to get product ID from URL
7739		- $product_id = url_to_postid($url);
7740		-
7741		- // If url_to_postid fails, try to extract from URL pattern
7742		- if (!$product_id) {
7743		- $product_slug = '';
7744		-
7745		- // Handle pretty permalinks: /product/product-name/
7746		- if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
7747		- $product_slug = $matches[1];
7748		- }
7749		-
7750		- if (!empty($product_slug)) {
7751		- $product_post = get_page_by_path($product_slug, OBJECT, 'product');
7752		- if ($product_post) {
7753		- $product_id = $product_post->ID;
7754		- }
7755		- }
7756		- }
7757		-
7758		- if (!$product_id) {
7759		- return false;
7760		- }
7761		-
7762		- // Get WooCommerce product object
7763		- $product = wc_get_product($product_id);
7764		-
7765		- if (!$product) {
7766		- return false;
7767		- }
7768		-
7769		- // Build product content with pricing (similar to mxchat_store_product_embedding)
7770		- $title = $product->get_name();
7771		- $description = $product->get_description();
7772		- $short_description = $product->get_short_description();
7773		- $sku = $product->get_sku();
7774		-
7775		- // Get pricing information
7776		- $regular_price = $product->get_regular_price();
7777		- $sale_price = $product->get_sale_price();
7778		- $price = $product->get_price(); // Current active price
7779		-
7780		- // Get currency symbol
7781		- $currency_symbol = get_woocommerce_currency_symbol();
7782		-
7783		- // Format content
7784		- $content = $title . "\n\n";
7785		-
7786		- if (!empty($short_description)) {
7787		- $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
7788		- }
7789		-
7790		- if (!empty($description)) {
7791		- $content .= wp_strip_all_tags($description) . "\n\n";
7792		- }
7793		-
7794		- // Add pricing information
7795		- if (!empty($regular_price)) {
7796		- $content .= "Price: " . $currency_symbol . $regular_price . "\n";
7797		- } elseif (!empty($price)) {
7798		- $content .= "Price: " . $currency_symbol . $price . "\n";
7799		- }
7800		-
7801		- if (!empty($sale_price) && $sale_price !== $regular_price) {
7802		- $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
7803		- }
7804		-
7805		- // Handle variable products - show price range
7806		- if ($product->is_type('variable')) {
7807		- $min_price = $product->get_variation_price('min');
7808		- $max_price = $product->get_variation_price('max');
7809		- if ($min_price !== $max_price) {
7810		- $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
7811		- }
7812		- }
7813		-
7814		- if (!empty($sku)) {
7815		- $content .= "SKU: " . $sku . "\n";
7816		- }
7817		-
7818		- // Get product categories
7819		- $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
7820		- if (!empty($categories) && !is_wp_error($categories)) {
7821		- $content .= "Categories: " . implode(', ', $categories) . "\n";
7822		- }
7823		-
7824		- // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
7825		- $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
7826		- if (!empty($custom_tabs) && is_array($custom_tabs)) {
7827		- foreach ($custom_tabs as $tab) {
7828		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
7829		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
7830		-
7831		- if (!empty($tab_title) && !empty($tab_content)) {
7832		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
7833		- }
7834		- }
7835		- }
7836		-
7837		- // Also check for reusable/saved tabs applied to this product
7838		- $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
7839		- if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
7840		- $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
7841		- if (!empty($saved_tabs) && is_array($saved_tabs)) {
7842		- foreach ($applied_saved_tabs as $saved_tab_id) {
7843		- if (isset($saved_tabs[$saved_tab_id])) {
7844		- $tab = $saved_tabs[$saved_tab_id];
7845		- $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
7846		- $tab_content = isset($tab['content']) ? $tab['content'] : '';
7847		-
7848		- if (!empty($tab_title) && !empty($tab_content)) {
7849		- $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
7850		- }
7851		- }
7852		- }
7853		- }
7854		- }
7855		-
7856		- return $this->mxchat_sanitize_content_for_api($content);
7857		-}
7858		-
7859		-/**
7860		- * Process a PDF page from the queue
7861		- */
7862		-private function mxchat_process_queue_pdf_page($item_data, $bot_id = 'default') {
7863		- $pdf_path = isset($item_data['pdf_path']) ? $item_data['pdf_path'] : '';
7864		- $pdf_url = isset($item_data['pdf_url']) ? $item_data['pdf_url'] : '';
7865		- $page_number = isset($item_data['page_number']) ? absint($item_data['page_number']) : 0;
7866		- $total_pages = isset($item_data['total_pages']) ? absint($item_data['total_pages']) : 0;
7867		-
7868		- if (empty($pdf_path) || !file_exists($pdf_path)) {
7869		- return new WP_Error('pdf_not_found', 'PDF file not found: ' . $pdf_path);
7870		- }
7871		-
7872		- if ($page_number < 1) {
7873		- return new WP_Error('invalid_page', 'Invalid page number');
7874		- }
7875		-
7876		- try {
7877		- mxchat_load_pdf_parser();
7878		- $parser = new \Smalot\PdfParser\Parser();
7879		- $pdf = $parser->parseFile($pdf_path);
7880		- $pages = $pdf->getPages();
7881		-
7882		- if (!isset($pages[$page_number - 1])) {
7883		- return new WP_Error('page_not_found', 'Page ' . $page_number . ' not found in PDF');
7884		- }
7885		-
7886		- $text = $pages[$page_number - 1]->getText();
7887		-
7888		- if (empty($text)) {
7889		- return new WP_Error('empty_page', 'Page ' . $page_number . ': No text could be extracted — page may contain only images, links, or non-standard encoding');
7890		- }
7891		-
7892		- $sanitized = $this->mxchat_sanitize_content_for_api($text);
7893		-
7894		- if (empty($sanitized)) {
7895		- return new WP_Error('empty_after_sanitization', 'Page ' . $page_number . ': Text was extracted but contained only special characters, control codes, or unsupported content that was removed during cleanup');
7896		- }
7897		-
7898		- // Create metadata
7899		- $metadata = array(
7900		- 'document_type' => 'pdf',
7901		- 'total_pages' => $total_pages,
7902		- 'current_page' => $page_number,
7903		- 'source_url' => $pdf_url
7904		- );
7905		-
7906		- $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
7907		- $page_url = esc_url($pdf_url . "#page=" . $page_number);
7908		-
7909		- // Get bot-specific API key
7910		- $bot_options = $this->get_bot_options($bot_id);
7911		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
7912		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
7913		-
7914		- if (strpos($selected_model, 'voyage') === 0) {
7915		- $api_key = $options['voyage_api_key'] ?? '';
7916		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
7917		- $api_key = $options['gemini_api_key'] ?? '';
7918		- } else {
7919		- $api_key = $options['api_key'] ?? '';
7920		- }
7921		-
7922		- if (empty($api_key)) {
7923		- return new WP_Error('no_api_key', 'API key not configured for bot: ' . $bot_id);
7924		- }
7925		-
7926		- // Submit to database - UPDATED 2.5.6: Added content_type 'pdf'
7927		- $result = MxChat_Utils::submit_content_to_db(
7928		- $content_with_metadata,
7929		- $page_url,
7930		- $api_key,
7931		- null,
7932		- $bot_id,
7933		- 'pdf'
7934		- );
7935		-
7936		- return $result;
7937		-
7938		- } catch (Exception $e) {
7939		- return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
7940		- }
7941		-}
7942		-
7943		-/**
7944		- * AJAX: Get queue processing status
7945		- */
7946		-public function ajax_mxchat_get_queue_status() {
7947		- // Verify nonce and permissions
7948		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
7949		-
7950		- if (!current_user_can('manage_options')) {
7951		- wp_send_json_error('Unauthorized access');
7952		- }
7953		-
7954		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7955		-
7956		- if (empty($queue_id)) {
7957		- wp_send_json_error('Missing queue ID');
7958		- }
7959		-
7960		- global $wpdb;
7961		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7962		-
7963		- // Get counts by status
7964		- $counts = $wpdb->get_results($wpdb->prepare(
7965		- "SELECT status, COUNT(*) as count
7966		- FROM $table_name
7967		- WHERE queue_id = %s
7968		- GROUP BY status",
7969		- $queue_id
7970		- ), OBJECT_K);
7971		-
7972		- $total = 0;
7973		- $completed = 0;
7974		- $failed = 0;
7975		- $processing = 0;
7976		- $pending = 0;
7977		-
7978		- foreach ($counts as $status => $data) {
7979		- $count = absint($data->count);
7980		- $total += $count;
7981		-
7982		- switch ($status) {
7983		- case 'completed':
7984		- $completed = $count;
7985		- break;
7986		- case 'failed':
7987		- $failed = $count;
7988		- break;
7989		- case 'processing':
7990		- $processing = $count;
7991		- break;
7992		- case 'pending':
7993		- $pending = $count;
7994		- break;
7995		- }
7996		- }
7997		-
7998		- // Calculate percentage
7999		- $percentage = $total > 0 ? round((($completed + $failed) / $total) * 100) : 0;
8000		-
8001		- // Get failed items details (include all failed items, not just those that exhausted retries)
8002		- $failed_items = array();
8003		- if ($failed > 0) {
8004		- $failed_items = $wpdb->get_results($wpdb->prepare(
8005		- "SELECT item_type, item_data, error_message, attempts
8006		- FROM $table_name
8007		- WHERE queue_id = %s
8008		- AND status = 'failed'
8009		- ORDER BY id DESC
8010		- LIMIT 50",
8011		- $queue_id
8012		- ));
8013		- }
8014		-
8015		- // Get queue metadata
8016		- $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
8017		- $queue_type = $this->mxchat_get_queue_meta($queue_id, 'queue_type');
8018		-
8019		- // Determine if queue is complete
8020		- $is_complete = ($pending === 0 && $processing === 0);
8021		-
8022		- wp_send_json_success(array(
8023		- 'queue_id' => $queue_id,
8024		- 'queue_type' => $queue_type,
8025		- 'source_url' => $source_url,
8026		- 'total' => $total,
8027		- 'completed' => $completed,
8028		- 'failed' => $failed,
8029		- 'processing' => $processing,
8030		- 'pending' => $pending,
8031		- 'percentage' => $percentage,
8032		- 'is_complete' => $is_complete,
8033		- 'failed_items' => $failed_items,
8034		- 'status' => $is_complete ? 'complete' : 'processing'
8035		- ));
8036		-}
8037		-
8038		-/**
8039		- * AJAX: Clear completed queue
8040		- */
8041		-public function ajax_mxchat_clear_queue() {
8042		- // Verify nonce and permissions
8043		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8044		-
8045		- if (!current_user_can('manage_options')) {
8046		- wp_send_json_error('Unauthorized access');
8047		- }
8048		-
8049		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8050		-
8051		- if (empty($queue_id)) {
8052		- wp_send_json_error('Missing queue ID');
8053		- }
8054		-
8055		- global $wpdb;
8056		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8057		- $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
8058		-
8059		- // Delete queue items
8060		- $wpdb->delete(
8061		- $table_name,
8062		- array('queue_id' => $queue_id),
8063		- array('%s')
8064		- );
8065		-
8066		- // Delete queue metadata
8067		- $wpdb->delete(
8068		- $meta_table,
8069		- array('queue_id' => $queue_id),
8070		- array('%s')
8071		- );
8072		-
8073		- wp_send_json_success(array(
8074		- 'message' => 'Queue cleared successfully'
8075		- ));
8076		-}
8077		-
8078		-/**
8079		- * AJAX: Retry failed items in queue
8080		- */
8081		-public function ajax_mxchat_retry_failed() {
8082		- // Verify nonce and permissions
8083		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8084		-
8085		- if (!current_user_can('manage_options')) {
8086		- wp_send_json_error('Unauthorized access');
8087		- }
8088		-
8089		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8090		-
8091		- if (empty($queue_id)) {
8092		- wp_send_json_error('Missing queue ID');
8093		- }
8094		-
8095		- global $wpdb;
8096		- $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8097		-
8098		- // Reset failed items to pending and reset attempt count
8099		- $updated = $wpdb->update(
8100		- $table_name,
8101		- array(
8102		- 'status' => 'pending',
8103		- 'attempts' => 0,
8104		- 'error_message' => null
8105		- ),
8106		- array(
8107		- 'queue_id' => $queue_id,
8108		- 'status' => 'failed'
8109		- ),
8110		- array('%s', '%d', '%s'),
8111		- array('%s', '%s')
8112		- );
8113		-
8114		- wp_send_json_success(array(
8115		- 'message' => 'Reset ' . $updated . ' failed items for retry',
8116		- 'reset_count' => $updated
8117		- ));
8118		-}
8119		-
8120		-
8121		-public function ajax_mxchat_mark_queue_complete() {
8122		- check_ajax_referer('mxchat_queue_nonce', 'nonce');
8123		-
8124		- if (!current_user_can('manage_options')) {
8125		- wp_send_json_error('Unauthorized access');
8126		- }
8127		-
8128		- $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8129		-
8130		- if (empty($queue_id)) {
8131		- wp_send_json_error('Missing queue ID');
8132		- }
8133		-
8134		- // Clear active queue transients
8135		- if (strpos($queue_id, 'sitemap_') === 0) {
8136		- delete_transient('mxchat_active_queue_sitemap');
8137		- } else if (strpos($queue_id, 'pdf_') === 0) {
8138		- delete_transient('mxchat_active_queue_pdf');
8139		- }
8140		-
8141		- wp_send_json_success(array('message' => 'Queue marked as complete'));
8142		-}
8143		-
8144	3346
8145	3347	// ========================================
8146	3348	// STATIC ACCESS METHODS
8147	3349	// ========================================