PluginProbe
MxChat – AI Chatbot & Content Generation for WordPress / 2.2.4
MxChat – AI Chatbot & Content Generation for WordPress v2.2.4
3.2.21 3.2.20 3.2.19 3.2.18 3.2.17 3.2.16 3.2.15 3.2.14 3.2.12 3.2.13 3.2.11 3.2.10 3.2.9 3.2.8 3.2.7 3.2.6 3.2.5 3.2.4 3.2.3 3.2.2 3.2.1 2.0.3 2.0.4 2.0.5 2.0.6 All 152 releases
← All changes | admin/class-knowledge-manager.php +1651 -7067 3.2.17 → 2.2.4 View file →
@@ -9,101 +9,55 @@
9 9 exit; // Exit if accessed directly
10 10 }
11 11
12 12 class MxChat_Knowledge_Manager {
13 -
13 +
14 14 private $options;
15 -
16 - // Post IDs whose vectors were already deleted by mxchat_handle_status_transition this
17 - // request, so the transient-based branch in mxchat_handle_post_update can skip the
18 - // redundant (idempotent but network-visible) second deletion.
19 - private $transition_deleted_posts = array();
20 -
21 - // Post IDs already INDEXED by mxchat_handle_status_transition's arrival edge this
22 - // request. Normal editor publishes fire transition_post_status first, then
23 - // post_updated — without this guard every editor publish would embed twice.
24 - private $transition_indexed_posts = array();
25 -
15 +
26 16 /**
27 17 * Constructor - Register hooks for content processing
28 18 */
29 -public function __construct() {
30 - $this->options = get_option('mxchat_options', array());
31 - $this->mxchat_init_hooks();
19 + public function __construct() {
20 + $this->options = get_option('mxchat_options', array());
21 + $this->mxchat_init_hooks();
22 + }
32 23
33 - $this->mxchat_init_role_hooks();
34 -}
35 -
36 -/**
37 - * Initialize WordPress hooks for content processing
38 - *
39 - */
40 -private function mxchat_init_hooks() {
41 - // Admin post handlers for form submissions
42 - add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
43 - add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
44 - add_action('admin_post_mxchat_submit_pdf_file', array($this, 'mxchat_handle_pdf_file_submission'));
45 - add_action('admin_post_mxchat_submit_youtube', array($this, 'mxchat_handle_youtube_submission'));
46 - add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
47 -
48 - // AJAX handlers for real-time processing and status updates
49 - add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
50 - add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status'));
51 - add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
52 - add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
53 - add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
54 - add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
55 - add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
56 - add_action('wp_ajax_mxchat_delete_chunks_by_url', array($this, 'ajax_mxchat_delete_chunks_by_url'));
57 - add_action('wp_ajax_mxchat_delete_wordpress_prompt', array($this, 'ajax_mxchat_delete_wordpress_prompt'));
58 - add_action('wp_ajax_mxchat_bulk_delete_knowledge', array($this, 'ajax_mxchat_bulk_delete_knowledge'));
59 - add_action('wp_ajax_mxchat_update_role_restriction', array($this, 'ajax_mxchat_update_role_restriction'));
60 -
61 - // Queue-based processing AJAX handlers
62 - add_action('wp_ajax_mxchat_get_next_queue_item', array($this, 'ajax_mxchat_get_next_queue_item'));
63 - add_action('wp_ajax_mxchat_process_queue_item', array($this, 'ajax_mxchat_process_queue_item'));
64 - add_action('wp_ajax_mxchat_get_queue_status', array($this, 'ajax_mxchat_get_queue_status'));
65 - add_action('wp_ajax_mxchat_clear_queue', array($this, 'ajax_mxchat_clear_queue'));
66 - add_action('wp_ajax_mxchat_retry_failed', array($this, 'ajax_mxchat_retry_failed'));
67 - add_action('wp_ajax_mxchat_get_recent_entries', array($this, 'ajax_mxchat_get_recent_entries'));
68 - add_action('wp_ajax_mxchat_detect_sitemaps', array($this, 'ajax_mxchat_detect_sitemaps'));
69 - add_action('wp_ajax_mxchat_refresh_pinecone_entries', array($this, 'ajax_mxchat_refresh_pinecone_entries'));
70 - add_action('wp_ajax_mxchat_paginate_entries', array($this, 'ajax_mxchat_paginate_entries'));
71 - add_action('wp_ajax_mxchat_get_entry_content', array($this, 'ajax_mxchat_get_entry_content'));
72 - add_action('wp_ajax_mxchat_save_entry_content', array($this, 'ajax_mxchat_save_entry_content'));
73 - add_action('wp_ajax_mxchat_inspect_entry', array($this, 'ajax_mxchat_inspect_entry'));
24 + /**
25 + * Initialize WordPress hooks for content processing
26 + */
27 + private function mxchat_init_hooks() {
28 + // Admin post handlers for form submissions
29 + add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
30 + add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
31 + add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
32 +
33 + // AJAX handlers for real-time processing and status updates
34 + add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
35 + add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status')); // NEW
36 + add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
37 + add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
38 + add_action('wp_ajax_mxchat_manual_batch_process', array($this, 'ajax_manual_batch_process'));
39 + add_action('mxchat_delete_content', array($this, 'mxchat_delete_from_pinecone_by_url'), 10, 1);
74 40
75 - // WordPress post management hooks
76 - add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
77 - add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
78 - add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
79 - add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
80 - // Authoritative unpublish detection: core hands this hook the REAL previous status, so
81 - // removal no longer depends on the mxchat_prev_status_* transients (evictable by persistent
82 - // object caches, never written by paths that bypass wp_update_post, e.g. plugins flipping
83 - // post_status directly and calling wp_transition_post_status themselves).
84 - add_action('transition_post_status', array($this, 'mxchat_handle_status_transition'), 10, 3);
41 +
42 + // Cron handlers for background processing
43 + add_action('mxchat_process_sitemap_urls', array($this, 'mxchat_process_sitemap_urls_cron'), 10, 5);
44 + add_action('mxchat_process_pdf_pages', array($this, 'mxchat_process_pdf_pages_cron'), 10, 5);
45 +
46 + // WordPress post management hooks
47 + add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
48 + add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
49 + add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
50 + add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
85 51
86 - // ACF hook - fires AFTER ACF fields are saved, ensuring ACF data is available
87 - // Priority 20 to run after ACF's own save (which runs at priority 10)
88 - add_action('acf/save_post', array($this, 'mxchat_handle_acf_save'), 20);
89 -
90 - // One-time cleanup for vectors orphaned by unpublishes that predate the
91 - // transition_post_status handler (plan 816fb1): wp mxchat prune-unpublished
92 - if (defined('WP_CLI') && WP_CLI) {
93 - WP_CLI::add_command('mxchat prune-unpublished', array($this, 'cli_prune_unpublished'));
52 + // WooCommerce product hooks (if WooCommerce is active)
53 + if (class_exists('WooCommerce')) {
54 + add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
55 + add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
56 + add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
57 + }
58 +
94 59 }
95 -
96 - add_action('wp_ajax_mxchat_mark_queue_complete', array($this, 'ajax_mxchat_mark_queue_complete'));
97 -
98 - // WooCommerce product hooks (if WooCommerce is active)
99 - if (class_exists('WooCommerce')) {
100 - add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
101 - add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
102 - add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
103 - add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
104 - }
105 -}
106 60
107 61 /**
108 62 * Get current options (refreshed)
109 63 */
@@ -113,9 +67,315 @@
113 67 }
114 68 return $this->options;
115 69 }
116 70
71 +
72 + /**
73 + * Handle manual batch processing via AJAX
74 + */
75 +public function ajax_manual_batch_process() {
76 + try {
77 + // Verify nonce and permissions
78 + check_ajax_referer('mxchat_status_nonce', 'nonce');
79 +
80 + if (!current_user_can('manage_options')) {
81 + wp_send_json_error('Unauthorized access');
82 + }
83 +
84 + $process_type = sanitize_text_field($_POST['process_type'] ?? '');
85 + $url = sanitize_text_field($_POST['url'] ?? '');
86 +
87 + if (empty($process_type) || empty($url)) {
88 + wp_send_json_error('Missing required parameters');
89 + }
90 +
91 + $processed = 0;
92 +
93 + if ($process_type === 'pdf') {
94 + $processed = $this->mxchat_manual_process_pdf_batch($url);
95 + } elseif ($process_type === 'sitemap') {
96 + $processed = $this->mxchat_manual_process_sitemap_batch($url);
97 + }
98 +
99 + if ($processed > 0) {
100 + wp_send_json_success(array(
101 + 'message' => "Processed {$processed} items successfully",
102 + 'processed' => $processed
103 + ));
104 + } else {
105 + wp_send_json_error('No items were processed');
106 + }
107 +
108 + } catch (Exception $e) {
109 + //error_log('Manual batch process error: ' . $e->getMessage());
110 + wp_send_json_error('Processing failed: ' . $e->getMessage());
111 + }
112 +}
117 113
114 +/**
115 + * Process a small PDF batch manually - DIRECT PROCESSING
116 + */
117 +private function mxchat_manual_process_pdf_batch($pdf_url) {
118 + try {
119 + $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
120 + $status = get_transient($status_key);
121 +
122 + if (!$status || $status['status'] !== 'processing') {
123 + //error_log('Manual PDF: No processing status found');
124 + return 0;
125 + }
126 +
127 + //error_log('Manual PDF: Starting direct processing for ' . $pdf_url);
128 +
129 + // Get current progress
130 + $current_page = $status['processed_pages'] ?? 0;
131 + $total_pages = $status['total_pages'] ?? 0;
132 +
133 + if ($current_page >= $total_pages) {
134 + //error_log('Manual PDF: Already completed');
135 + return 0;
136 + }
137 +
138 + // Try to download the PDF again for processing
139 + $response = wp_remote_get($pdf_url, array('timeout' => 30));
140 +
141 + if (is_wp_error($response)) {
142 + //error_log('Manual PDF: Failed to download PDF: ' . $response->get_error_message());
143 + return 0;
144 + }
145 +
146 + $pdf_content = wp_remote_retrieve_body($response);
147 + if (empty($pdf_content)) {
148 + //error_log('Manual PDF: Empty PDF content');
149 + return 0;
150 + }
151 +
152 + // Save PDF temporarily
153 + $upload_dir = wp_upload_dir();
154 + $temp_pdf_path = trailingslashit($upload_dir['path']) . 'temp_manual_' . time() . '.pdf';
155 + file_put_contents($temp_pdf_path, $pdf_content);
156 +
157 + // Process 2 pages directly
158 + $processed = $this->mxchat_process_pdf_pages_direct($temp_pdf_path, $pdf_url, $current_page, 5);
159 +
160 + // Clean up temp file
161 + if (file_exists($temp_pdf_path)) {
162 + wp_delete_file($temp_pdf_path);
163 + }
164 +
165 + //error_log('Manual PDF: Processed ' . $processed . ' pages');
166 + return $processed;
167 +
168 + } catch (Exception $e) {
169 + //error_log('Manual PDF batch error: ' . $e->getMessage());
170 + return 0;
171 + }
172 +}
173 +
174 +/**
175 + * Process PDF pages directly without cron
176 + */
177 +private function mxchat_process_pdf_pages_direct($pdf_path, $pdf_url, $start_page, $batch_size) {
178 + try {
179 + if (!file_exists($pdf_path)) {
180 + //error_log('Direct PDF: File not found at ' . $pdf_path);
181 + return 0;
182 + }
183 +
184 + $parser = new \Smalot\PdfParser\Parser();
185 + $pdf = $parser->parseFile($pdf_path);
186 + $pages = $pdf->getPages();
187 +
188 + $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
189 + $status = get_transient($status_key);
190 +
191 + if (!$status) {
192 + return 0;
193 + }
194 +
195 + $options = get_option('mxchat_options');
196 + $api_key = $options['api_key'] ?? '';
197 +
198 + if (empty($api_key)) {
199 + //error_log('Direct PDF: No API key');
200 + return 0;
201 + }
202 +
203 + $processed = 0;
204 + $end_page = min($start_page + $batch_size, count($pages));
205 +
206 + for ($i = $start_page; $i < $end_page; $i++) {
207 + try {
208 + $page_number = $i + 1;
209 + $text = $pages[$i]->getText();
210 +
211 + if (empty($text)) {
212 + //error_log('Direct PDF: Empty text on page ' . $page_number);
213 + continue;
214 + }
215 +
216 + $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
217 + if (empty($sanitized_content)) {
218 + //error_log('Direct PDF: No content after sanitization on page ' . $page_number);
219 + continue;
220 + }
221 +
222 + // Generate embedding
223 + $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
224 + if (is_string($embedding_vector)) {
225 + //error_log('Direct PDF: Embedding failed on page ' . $page_number . ': ' . $embedding_vector);
226 + continue;
227 + }
228 +
229 + // Create metadata
230 + $metadata = array(
231 + 'document_type' => 'pdf',
232 + 'total_pages' => count($pages),
233 + 'current_page' => $page_number,
234 + 'source_url' => $pdf_url
235 + );
236 +
237 + $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
238 + $page_url = esc_url($pdf_url . "#page=" . $page_number);
239 +
240 + // Store in database
241 + $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $api_key);
242 +
243 + if (is_wp_error($db_result)) {
244 + //error_log('Direct PDF: DB error on page ' . $page_number . ': ' . $db_result->get_error_message());
245 + continue;
246 + }
247 +
248 + $processed++;
249 + //error_log('Direct PDF: Successfully processed page ' . $page_number);
250 +
251 + // Update status
252 + $status['processed_pages'] = $i + 1;
253 + $status['last_update'] = time();
254 + $status['percentage'] = round(($status['processed_pages'] / $status['total_pages']) * 100);
255 + set_transient($status_key, $status, DAY_IN_SECONDS);
256 +
257 + } catch (Exception $e) {
258 + //error_log('Direct PDF: Error processing page ' . ($i + 1) . ': ' . $e->getMessage());
259 + continue;
260 + }
261 + }
262 +
263 + // Check if completed
264 + if ($status['processed_pages'] >= $status['total_pages']) {
265 + $status['status'] = 'complete';
266 + set_transient($status_key, $status, DAY_IN_SECONDS);
267 + //error_log('Direct PDF: Processing completed');
268 + }
269 +
270 + return $processed;
271 +
272 + } catch (Exception $e) {
273 + //error_log('Direct PDF processing error: ' . $e->getMessage());
274 + return 0;
275 + }
276 +}
277 +
278 +/**
279 + * Process a small sitemap batch manually - DIRECT PROCESSING
280 + */
281 +private function mxchat_manual_process_sitemap_batch($sitemap_url) {
282 + try {
283 + $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
284 + $status = get_transient($status_key);
285 +
286 + if (!$status || $status['status'] !== 'processing') {
287 + return 0;
288 + }
289 +
290 + //error_log('Manual Sitemap: Starting direct processing for ' . $sitemap_url);
291 +
292 + // Re-fetch the sitemap to get URLs
293 + $response = wp_remote_get($sitemap_url, array('timeout' => 30));
294 + if (is_wp_error($response)) {
295 + //error_log('Manual Sitemap: Failed to fetch sitemap');
296 + return 0;
297 + }
298 +
299 + $sitemap_content = wp_remote_retrieve_body($response);
300 + $xml = simplexml_load_string($sitemap_content);
301 +
302 + if (!$xml) {
303 + //error_log('Manual Sitemap: Invalid XML');
304 + return 0;
305 + }
306 +
307 + $urls = array();
308 + foreach ($xml->url as $url_element) {
309 + $urls[] = (string)$url_element->loc;
310 + }
311 +
312 + $current_processed = $status['processed_urls'] ?? 0;
313 + $batch_size = 5;
314 + $processed = 0;
315 +
316 + // Process next 2 URLs
317 + for ($i = $current_processed; $i < min($current_processed + $batch_size, count($urls)); $i++) {
318 + $url = $urls[$i];
319 +
320 + if ($this->mxchat_process_single_url_direct($url)) {
321 + $processed++;
322 + }
323 +
324 + // Update status
325 + $status['processed_urls'] = $i + 1;
326 + $status['last_update'] = time();
327 + $status['percentage'] = round(($status['processed_urls'] / $status['total_urls']) * 100);
328 + set_transient($status_key, $status, DAY_IN_SECONDS);
329 + }
330 +
331 + // Check if completed
332 + if ($status['processed_urls'] >= $status['total_urls']) {
333 + $status['status'] = 'complete';
334 + set_transient($status_key, $status, DAY_IN_SECONDS);
335 + }
336 +
337 + //error_log('Manual Sitemap: Processed ' . $processed . ' URLs');
338 + return $processed;
339 +
340 + } catch (Exception $e) {
341 + //error_log('Manual sitemap batch error: ' . $e->getMessage());
342 + return 0;
343 + }
344 +}
345 +
346 +/**
347 + * Process a single URL directly
348 + */
349 +private function mxchat_process_single_url_direct($url) {
350 + try {
351 + $response = wp_remote_get($url, array('timeout' => 30));
352 + if (is_wp_error($response)) {
353 + return false;
354 + }
355 +
356 + $html = wp_remote_retrieve_body($response);
357 + $content = $this->mxchat_extract_main_content($html);
358 + $sanitized = $this->mxchat_sanitize_content_for_api($content);
359 +
360 + if (empty($sanitized)) {
361 + return false;
362 + }
363 +
364 + $options = get_option('mxchat_options');
365 + $api_key = $options['api_key'] ?? '';
366 +
367 + $result = MxChat_Utils::submit_content_to_db($sanitized, $url, $api_key);
368 +
369 + return !is_wp_error($result);
370 +
371 + } catch (Exception $e) {
372 + //error_log('Single URL processing error: ' . $e->getMessage());
373 + return false;
374 + }
375 +}
376 +
377 +
118 378 // ========================================
119 379 // MAIN CONTENT SUBMISSION HANDLERS
120 380 // ========================================
121 381
@@ -131,30 +391,34 @@
131 391 wp_die(esc_html__('Nonce verification failed.', 'mxchat'));
132 392 }
133 393
134 394 // Sanitize the inputs.
135 - // Use wp_kses_post to allow safe HTML and wp_unslash to remove WordPress added slashes
136 - $article_content = wp_kses_post(wp_unslash($_POST['article_content']));
395 + $article_content = sanitize_textarea_field($_POST['article_content']);
137 396 $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
138 397
139 - // Get bot_id from form submission
140 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
398 + // Get API key for submission
399 + $options = get_option('mxchat_options');
400 + $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
141 401
142 - // Get bot-specific options and API key
143 - $bot_options = $this->get_bot_options($bot_id);
144 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
145 -
146 - // Custom-provider-aware decision; keyless custom sites must pass (plan cbd5fd).
147 - $preflight = MxChat_Utils::embedding_preflight($options);
148 - if (!$preflight['ok']) {
149 - set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
402 + if (strpos($selected_model, 'voyage') === 0) {
403 + $api_key = $options['voyage_api_key'] ?? '';
404 + } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
405 + $api_key = $options['gemini_api_key'] ?? '';
406 + } else {
407 + $api_key = $options['api_key'] ?? '';
408 + }
409 +
410 + if (empty($api_key)) {
411 + set_transient('mxchat_admin_notice_error',
412 + esc_html__('API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
413 + 30
414 + );
150 415 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
151 416 exit;
152 417 }
153 - $api_key = $preflight['api_key'];
154 418
155 - // Use centralized utility function with bot_id
156 - $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key, null, $bot_id);
419 + // Use centralized utility function for storage
420 + $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key);
157 421
158 422 if (is_wp_error($result)) {
159 423 set_transient('mxchat_admin_notice_error',
160 424 esc_html__('Error storing content: ', 'mxchat') . $result->get_error_message(),
@@ -169,253 +433,17 @@
169 433
170 434 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
171 435 exit;
172 436 }
173 -
174 -/**
175 - * Handle the "YouTube" KB import source (admin-post form submission).
176 - *
177 - * Per-video description mode:
178 - * - auto: fetch oEmbed metadata (reliable) + best-effort captions transcript.
179 - * If no usable transcript, index the metadata anyway, tell the admin,
180 - * and bounce back with the manual box pre-filled (never fail silently).
181 - * - manual: the admin's own description is what gets indexed; metadata rides along.
182 - *
183 - * The row is stored with content_type 'youtube' and source_url = the canonical
184 - * watch URL, so re-importing the same video UPDATES the entry (source_url
185 - * duplicate handling in MxChat_Utils::store_in_wordpress_db) — that is also the
186 - * "augment a metadata-only entry" path.
187 - */
188 -public function mxchat_handle_youtube_submission() {
189 - if (!isset($_POST['submit_youtube']) || !current_user_can('manage_options')) {
190 - wp_die(esc_html__('Unauthorized access', 'mxchat'));
191 - }
192 -
193 - check_admin_referer('mxchat_submit_youtube_action', 'mxchat_submit_youtube_nonce');
194 -
195 - $redirect_url = admin_url('admin.php?page=mxchat-prompts');
196 -
197 - $youtube_url = isset($_POST['youtube_url']) ? esc_url_raw(wp_unslash($_POST['youtube_url'])) : '';
198 - $video_id = MxChat_Utils::parse_youtube_id($youtube_url);
199 -
200 - if (empty($video_id)) {
201 - set_transient('mxchat_admin_notice_error',
202 - esc_html__('That does not look like a link to a single YouTube video. Please paste a watch, youtu.be, or Shorts URL.', 'mxchat'),
203 - 30
204 - );
205 - wp_safe_redirect(esc_url($redirect_url));
206 - exit;
207 - }
208 -
209 - $canonical_url = 'https://www.youtube.com/watch?v=' . $video_id;
210 -
211 - $description_mode = (isset($_POST['youtube_description_mode']) && $_POST['youtube_description_mode'] === 'manual') ? 'manual' : 'auto';
212 - $manual_description = isset($_POST['youtube_description']) ? trim(wp_kses_post(wp_unslash($_POST['youtube_description']))) : '';
213 -
214 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
215 -
216 - // Resolve the embedding decision exactly like the sibling handlers —
217 - // custom-provider-aware (plan cbd5fd).
218 - $bot_options = $this->get_bot_options($bot_id);
219 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
220 -
221 - $preflight = MxChat_Utils::embedding_preflight($options);
222 - if (!$preflight['ok']) {
223 - set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
224 - wp_safe_redirect(esc_url($redirect_url));
225 - exit;
226 - }
227 - $api_key = $preflight['api_key'];
228 -
229 - // Metadata is fetched in BOTH modes — it is the reliable half of auto, and in
230 - // manual mode it enriches the indexed text with the real title/channel.
231 - $meta = $this->mxchat_fetch_youtube_oembed($video_id);
232 - $video_title = isset($meta['title']) ? sanitize_text_field($meta['title']) : '';
233 - $video_channel = isset($meta['author_name']) ? sanitize_text_field($meta['author_name']) : '';
234 -
235 - $header_lines = 'YouTube Video: ' . ($video_title !== '' ? $video_title : $canonical_url) . "\n";
236 - if ($video_channel !== '') {
237 - $header_lines .= 'Channel: ' . $video_channel . "\n";
238 - }
239 - $header_lines .= 'URL: ' . $canonical_url . "\n\n";
240 -
241 - $transcript_missing = false;
242 -
243 - if ($description_mode === 'manual') {
244 - if ($manual_description === '') {
245 - set_transient('mxchat_admin_notice_error',
246 - esc_html__('Please write a description for the video, or switch to auto-fetch.', 'mxchat'),
247 - 30
248 - );
249 - wp_safe_redirect(esc_url($redirect_url));
250 - exit;
251 - }
252 - $indexed_text = $header_lines . $manual_description;
253 - } else {
254 - $transcript = $this->mxchat_fetch_youtube_transcript($video_id);
255 -
256 - if (strlen($transcript) >= 200) {
257 - $indexed_text = $header_lines . $transcript;
258 - } else {
259 - // Graceful fallback: captions disabled / blocked / no speech. Auto
260 - // reliably gets metadata; it does NOT guarantee a transcript.
261 - $transcript_missing = true;
262 -
263 - if ($video_title === '' && $video_channel === '') {
264 - // Both halves failed — nothing meaningful to index.
265 - set_transient('mxchat_admin_notice_error',
266 - esc_html__('Could not retrieve any information for that video (no metadata and no captions). Please check the URL, or use the manual description option.', 'mxchat'),
267 - 30
268 - );
269 - wp_safe_redirect(esc_url($redirect_url));
270 - exit;
271 - }
272 -
273 - $indexed_text = $header_lines . sprintf(
274 - /* translators: 1: video title, 2: channel name */
275 - __('A YouTube video titled "%1$s" from the channel %2$s.', 'mxchat'),
276 - $video_title !== '' ? $video_title : $canonical_url,
277 - $video_channel !== '' ? $video_channel : 'YouTube'
278 - );
279 - }
280 - }
281 -
282 - $result = MxChat_Utils::submit_content_to_db($indexed_text, $canonical_url, $api_key, null, $bot_id, 'youtube');
283 -
284 - if (is_wp_error($result)) {
285 - set_transient('mxchat_admin_notice_error',
286 - esc_html__('Error storing video in the knowledge base: ', 'mxchat') . $result->get_error_message(),
287 - 30
288 - );
289 - wp_safe_redirect(esc_url($redirect_url));
290 - exit;
291 - }
292 -
293 - if ($transcript_missing) {
294 - set_transient('mxchat_admin_notice_success',
295 - esc_html__('Video indexed from its title and channel — no captions were available for a transcript. The form below is pre-filled: write your own description and import again to improve matching (it updates the same entry).', 'mxchat'),
296 - 30
297 - );
298 - // Bounce back with prefill args so the page reopens the YouTube form in
299 - // manual mode with the URL + fetched title ready to augment.
300 - $redirect_url = add_query_arg(array(
301 - 'mxchat_yt_prefill' => '1',
302 - 'yt_url' => rawurlencode($canonical_url),
303 - 'yt_title' => rawurlencode($video_title),
304 - ), $redirect_url);
305 - } else {
306 - set_transient('mxchat_admin_notice_success',
307 - esc_html__('YouTube video successfully added to the knowledge base!', 'mxchat'),
308 - 30
309 - );
310 - }
311 -
312 - wp_safe_redirect(esc_url_raw($redirect_url));
313 - exit;
314 -}
315 -
316 -/**
317 - * Fetch YouTube oEmbed metadata for a video (no API key required).
318 - * Returns the decoded array (title, author_name, thumbnail_url, ...) or array().
319 - */
320 -private function mxchat_fetch_youtube_oembed($video_id) {
321 - $oembed_url = 'https://www.youtube.com/oembed?url=' . rawurlencode('https://www.youtube.com/watch?v=' . $video_id) . '&format=json';
322 - $response = wp_remote_get($oembed_url, array('timeout' => 15));
323 - if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
324 - return array();
325 - }
326 - $data = json_decode(wp_remote_retrieve_body($response), true);
327 - return is_array($data) ? $data : array();
328 -}
329 -
330 -/**
331 - * Best-effort captions transcript for a video. Deliberately ISOLATED: this uses
332 - * YouTube's unofficial timedtext route (the caption track list embedded in the
333 - * watch page), which YouTube has broken before and will break again. Every
334 - * failure mode returns '' so a break degrades to the metadata-only import path
335 - * instead of erroring the whole submission. Do not let anything in here throw.
336 - */
337 -private function mxchat_fetch_youtube_transcript($video_id) {
338 - $watch_url = 'https://www.youtube.com/watch?v=' . $video_id . '&hl=en';
339 -
340 - // First try the honest ingest UA; some responses omit the player config for
341 - // bot UAs, so retry once with a browser UA before giving up.
342 - $user_agents = array(
343 - mxchat_ingest_user_agent(),
344 - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
345 - );
346 -
347 - $tracks = array();
348 - foreach ($user_agents as $ua) {
349 - $response = wp_remote_get($watch_url, array(
350 - 'timeout' => 20,
351 - 'user-agent' => $ua,
352 - ));
353 - if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
354 - continue;
355 - }
356 - $body = wp_remote_retrieve_body($response);
357 - if (!is_string($body) || $body === '' || !preg_match('/"captionTracks":(\[.*?\])(?=,")/s', $body, $m)) {
358 - continue;
359 - }
360 - $decoded = json_decode($m[1], true);
361 - if (is_array($decoded) && !empty($decoded)) {
362 - $tracks = $decoded;
363 - break;
364 - }
365 - }
366 -
367 - if (empty($tracks)) {
368 - return '';
369 - }
370 -
371 - // Prefer an English track, else take the first offered.
372 - $chosen = null;
373 - foreach ($tracks as $track) {
374 - if (isset($track['languageCode']) && strpos($track['languageCode'], 'en') === 0) {
375 - $chosen = $track;
376 - break;
377 - }
378 - }
379 - if ($chosen === null) {
380 - $chosen = $tracks[0];
381 - }
382 - if (empty($chosen['baseUrl']) || !is_string($chosen['baseUrl'])) {
383 - return '';
384 - }
385 -
386 - $timedtext = wp_remote_get($chosen['baseUrl'], array('timeout' => 20));
387 - if (is_wp_error($timedtext) || wp_remote_retrieve_response_code($timedtext) !== 200) {
388 - return '';
389 - }
390 - $xml = wp_remote_retrieve_body($timedtext);
391 - if (!is_string($xml) || strpos($xml, '<text') === false) {
392 - return '';
393 - }
394 -
395 - // <text start=".." dur="..">caption</text> — strip tags, decode the
396 - // double-encoded entities timedtext ships, collapse whitespace.
397 - $text = preg_replace('/<[^>]+>/', ' ', $xml);
398 - $text = html_entity_decode(html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8'), ENT_QUOTES | ENT_HTML5, 'UTF-8');
399 - $text = trim(preg_replace('/\s+/u', ' ', $text));
400 -
401 - return $text;
402 -}
403 -
404 437 public function mxchat_is_pdf_url($url, $response) {
405 438 $content_type = wp_remote_retrieve_header($response, 'content-type');
406 439 $file_extension = strtolower(pathinfo($url, PATHINFO_EXTENSION));
407 440
408 - // Check Content-Disposition header for .pdf filename (Google Drive sends this)
409 - $disposition = wp_remote_retrieve_header($response, 'content-disposition');
410 - $has_pdf_disposition = ! empty($disposition) && stripos($disposition, '.pdf') !== false;
411 -
412 - return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf' || $has_pdf_disposition;
441 + return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf';
413 442 }
414 -
415 -
416 -public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response, $bot_id = 'default') {
443 +public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response) {
417 444 if (!current_user_can('manage_options')) {
445 + //error_log(esc_html__('Unauthorized PDF processing attempt', 'mxchat'));
418 446 return false;
419 447 }
420 448
421 449 $pdf_url = esc_url_raw($pdf_url);
@@ -421,8 +449,9 @@
421 449 $pdf_url = esc_url_raw($pdf_url);
422 450 $upload_dir = wp_upload_dir();
423 451
424 452 if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
453 + //error_log(sprintf(esc_html__('Upload directory error: %s', 'mxchat'), esc_html($upload_dir['error'])));
425 454 return false;
426 455 }
427 456
428 457 $pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
@@ -429,12 +458,14 @@
429 458 $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
430 459
431 460 $response_body = wp_remote_retrieve_body($response);
432 461 if (empty($response_body)) {
462 + //error_log(esc_html__('Empty PDF response body', 'mxchat'));
433 463 return false;
434 464 }
435 465
436 466 if (!wp_mkdir_p(dirname($pdf_path))) {
467 + //error_log(sprintf(esc_html__('Failed to create directory for PDF: %s', 'mxchat'), esc_html($pdf_path)));
437 468 return false;
438 469 }
439 470
440 471 try {
@@ -443,915 +474,295 @@
443 474 if (!file_exists($pdf_path)) {
444 475 throw new Exception(__('Failed to save PDF file', 'mxchat'));
445 476 }
446 477
447 - $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
448 -
449 - if ($total_pages === false || $total_pages < 1) {
450 - throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
451 - }
478 + $parser = new \Smalot\PdfParser\Parser();
479 + $pdf = $parser->parseFile($pdf_path);
480 + $total_pages = absint(count($pdf->getPages()));
452 481
453 - // Create unique queue ID
454 - $queue_id = 'pdf_' . md5($pdf_url . time());
455 -
456 - // Create array of pages to process
457 - $pages = array();
458 - for ($i = 1; $i <= $total_pages; $i++) {
459 - $pages[] = array(
460 - 'pdf_path' => $pdf_path,
461 - 'pdf_url' => $pdf_url,
462 - 'page_number' => $i,
463 - 'total_pages' => $total_pages
464 - );
482 + if ($total_pages < 1) {
483 + throw new Exception(__('Invalid PDF: no pages found', 'mxchat'));
465 484 }
466 485
467 - // Add pages to queue
468 - $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
486 + wp_schedule_single_event(time(), 'mxchat_process_pdf_pages', array(
487 + 'pdf_path' => $pdf_path,
488 + 'pdf_url' => $pdf_url,
489 + 'total_pages' => $total_pages,
490 + 'batch_size' => absint(15),
491 + 'batch_pause' => absint(10)
492 + ));
469 493
470 - if ($queued_count === 0) {
471 - wp_delete_file($pdf_path);
472 - throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
473 - }
494 + $status_data = array(
495 + 'total_pages' => $total_pages,
496 + 'processed_pages' => 0,
497 + 'status' => 'processing',
498 + 'last_update' => time()
499 + );
474 500
475 - // Store queue metadata
476 - $this->mxchat_set_queue_meta($queue_id, 'source_url', $pdf_url);
477 - $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
478 - $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
479 - $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
480 - $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
481 - $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
501 + set_transient(
502 + sanitize_key('mxchat_pdf_status_' . md5($pdf_url)),
503 + array_map('sanitize_text_field', $status_data),
504 + DAY_IN_SECONDS
505 + );
482 506
483 - // Store queue ID in transient for status tracking
484 - set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
485 - set_transient('mxchat_last_pdf_url', $pdf_url, DAY_IN_SECONDS);
507 + return __('scheduled', 'mxchat');
486 508
487 - return 'queued';
488 -
489 509 } catch (Exception $e) {
510 + //error_log(sprintf(esc_html__('Error preparing PDF for processing: %s', 'mxchat'), esc_html($e->getMessage())));
490 511 if (file_exists($pdf_path)) {
491 512 wp_delete_file($pdf_path);
492 513 }
493 - return $e->getMessage();
514 + return false;
494 515 }
495 516 }
496 517
497 -/**
498 - * Handle direct PDF file upload from the knowledge base page
499 - */
500 -public function mxchat_handle_pdf_file_submission() {
501 - if (!isset($_POST['submit_pdf_file']) || !current_user_can('manage_options')) {
502 - wp_die(esc_html__('Unauthorized access', 'mxchat'));
503 - }
518 +public function mxchat_process_pdf_pages_cron($pdf_path, $pdf_url, $total_pages, $batch_size, $batch_pause) {
519 + // Validate inputs
520 + $pdf_path = sanitize_text_field($pdf_path);
521 + $pdf_url = esc_url_raw($pdf_url);
522 + $total_pages = absint($total_pages);
523 + $batch_size = absint($batch_size);
524 + $batch_pause = absint($batch_pause);
504 525
505 - check_admin_referer('mxchat_submit_pdf_file_action', 'mxchat_submit_pdf_file_nonce');
506 -
507 - $redirect_url = admin_url('admin.php?page=mxchat-prompts');
508 -
509 - // Validate file upload
510 - if (empty($_FILES['pdf_file']) || $_FILES['pdf_file']['error'] !== UPLOAD_ERR_OK) {
511 - $error_code = isset($_FILES['pdf_file']['error']) ? $_FILES['pdf_file']['error'] : UPLOAD_ERR_NO_FILE;
512 - $error_messages = array(
513 - UPLOAD_ERR_INI_SIZE => __('The uploaded file exceeds the server upload_max_filesize limit.', 'mxchat'),
514 - UPLOAD_ERR_FORM_SIZE => __('The uploaded file exceeds the form MAX_FILE_SIZE limit.', 'mxchat'),
515 - UPLOAD_ERR_PARTIAL => __('The file was only partially uploaded.', 'mxchat'),
516 - UPLOAD_ERR_NO_FILE => __('No file was uploaded. Please select a PDF file.', 'mxchat'),
517 - UPLOAD_ERR_NO_TMP_DIR => __('Server missing temporary folder.', 'mxchat'),
518 - UPLOAD_ERR_CANT_WRITE => __('Server failed to write file to disk.', 'mxchat'),
519 - );
520 - $error_msg = isset($error_messages[$error_code]) ? $error_messages[$error_code] : __('Unknown upload error.', 'mxchat');
521 - set_transient('mxchat_admin_notice_error', $error_msg, 30);
522 - wp_safe_redirect(esc_url($redirect_url));
523 - exit;
524 - }
525 -
526 - $file = $_FILES['pdf_file'];
527 -
528 - // Validate MIME type
529 - $finfo = finfo_open(FILEINFO_MIME_TYPE);
530 - $mime_type = finfo_file($finfo, $file['tmp_name']);
531 - finfo_close($finfo);
532 -
533 - if ($mime_type !== 'application/pdf') {
534 - set_transient('mxchat_admin_notice_error',
535 - esc_html__('Invalid file type. Only PDF files are accepted.', 'mxchat'),
536 - 30
537 - );
538 - wp_safe_redirect(esc_url($redirect_url));
539 - exit;
540 - }
541 -
542 - // Validate extension
543 - $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
544 - if ($ext !== 'pdf') {
545 - set_transient('mxchat_admin_notice_error',
546 - esc_html__('Invalid file extension. Only .pdf files are accepted.', 'mxchat'),
547 - 30
548 - );
549 - wp_safe_redirect(esc_url($redirect_url));
550 - exit;
551 - }
552 -
553 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
554 - $original_filename = sanitize_file_name($file['name']);
555 -
556 - $upload_dir = wp_upload_dir();
557 - if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
558 - set_transient('mxchat_admin_notice_error',
559 - esc_html__('WordPress upload directory is not writable.', 'mxchat'),
560 - 30
561 - );
562 - wp_safe_redirect(esc_url($redirect_url));
563 - exit;
564 - }
565 -
566 - $pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
567 - $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
568 -
569 - if (!wp_mkdir_p(dirname($pdf_path))) {
570 - set_transient('mxchat_admin_notice_error',
571 - esc_html__('Failed to create upload directory.', 'mxchat'),
572 - 30
573 - );
574 - wp_safe_redirect(esc_url($redirect_url));
575 - exit;
576 - }
577 -
578 - // Move uploaded file
579 - if (!move_uploaded_file($file['tmp_name'], $pdf_path)) {
580 - set_transient('mxchat_admin_notice_error',
581 - esc_html__('Failed to save uploaded PDF file.', 'mxchat'),
582 - 30
583 - );
584 - wp_safe_redirect(esc_url($redirect_url));
585 - exit;
586 - }
587 -
588 526 try {
589 - $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
590 -
591 - if ($total_pages === false || $total_pages < 1) {
592 - throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
527 + if (!file_exists($pdf_path)) {
528 + throw new Exception(sprintf('PDF file not found at path: %s', esc_html($pdf_path)));
593 529 }
594 530
595 - // Use original filename as the source identifier
596 - $source_label = 'upload://' . $original_filename;
597 -
598 - $queue_id = 'pdf_' . md5($source_label . time());
599 -
600 - $pages = array();
601 - for ($i = 1; $i <= $total_pages; $i++) {
602 - $pages[] = array(
603 - 'pdf_path' => $pdf_path,
604 - 'pdf_url' => $source_label,
605 - 'page_number' => $i,
606 - 'total_pages' => $total_pages,
607 - );
608 - }
609 -
610 - $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
611 -
612 - if ($queued_count === 0) {
613 - wp_delete_file($pdf_path);
614 - throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
615 - }
616 -
617 - $this->mxchat_set_queue_meta($queue_id, 'source_url', $source_label);
618 - $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
619 - $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
620 - $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
621 - $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
622 - $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
623 -
624 - set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
625 - set_transient('mxchat_last_pdf_url', $source_label, DAY_IN_SECONDS);
626 -
627 - set_transient('mxchat_admin_notice_success',
628 - sprintf(
629 - esc_html__('PDF "%s" (%d pages) queued for processing. Processing will start automatically.', 'mxchat'),
630 - esc_html($original_filename),
631 - $total_pages
632 - ),
633 - 30
634 - );
635 -
636 - } catch (Exception $e) {
637 - if (file_exists($pdf_path)) {
638 - wp_delete_file($pdf_path);
639 - }
640 - set_transient('mxchat_admin_notice_error',
641 - esc_html__('Failed to process uploaded PDF: ', 'mxchat') . esc_html($e->getMessage()),
642 - 30
643 - );
644 - }
645 -
646 - wp_safe_redirect(esc_url($redirect_url));
647 - exit;
648 -}
649 -
650 -/**
651 - * Validate PDF and count pages with multiple parser attempts
652 - */
653 -private function mxchat_validate_and_count_pdf_pages($pdf_path) {
654 - // Method 1: Try with Smalot PDF Parser (your current method)
655 - try {
656 - mxchat_load_pdf_parser();
657 531 $parser = new \Smalot\PdfParser\Parser();
658 532 $pdf = $parser->parseFile($pdf_path);
659 533 $pages = $pdf->getPages();
660 - $page_count = count($pages);
661 -
662 - if ($page_count > 0) {
663 - //error_log('PDF parsed successfully with Smalot parser: ' . $page_count . ' pages');
664 - return $page_count;
665 - }
666 - } catch (Exception $e) {
667 - //error_log('Smalot PDF parser failed: ' . $e->getMessage());
668 - }
669 534
670 - // Method 2: Try with pdfinfo command (if available)
671 - if (function_exists('shell_exec') && !$this->mxchat_is_shell_disabled()) {
672 - try {
673 - $command = 'pdfinfo ' . escapeshellarg($pdf_path) . ' 2>&1';
674 - $output = shell_exec($command);
675 -
676 - if ($output && preg_match('/Pages:\s*(\d+)/', $output, $matches)) {
677 - $page_count = intval($matches[1]);
678 - if ($page_count > 0) {
679 - //error_log('PDF parsed successfully with pdfinfo: ' . $page_count . ' pages');
680 - return $page_count;
681 - }
682 - }
683 - } catch (Exception $e) {
684 - //error_log('pdfinfo command failed: ' . $e->getMessage());
685 - }
686 - }
535 + // Get current progress
536 + $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
537 + $status = get_transient($status_key);
687 538
688 - // Method 3: Try to repair PDF and parse again
689 - try {
690 - $repaired_path = $this->mxchat_attempt_pdf_repair($pdf_path);
691 - if ($repaired_path && $repaired_path !== $pdf_path) {
692 - mxchat_load_pdf_parser();
693 - $parser = new \Smalot\PdfParser\Parser();
694 - $pdf = $parser->parseFile($repaired_path);
695 - $pages = $pdf->getPages();
696 - $page_count = count($pages);
697 -
698 - if ($page_count > 0) {
699 - // Replace original with repaired version
700 - copy($repaired_path, $pdf_path);
701 - unlink($repaired_path);
702 - //error_log('PDF repaired and parsed successfully: ' . $page_count . ' pages');
703 - return $page_count;
704 - }
705 -
706 - // Clean up repaired file if it didn't work
707 - unlink($repaired_path);
539 + if (!$status || !is_array($status)) {
540 + throw new Exception('Invalid status data retrieved from transient');
708 541 }
709 - } catch (Exception $e) {
710 - //error_log('PDF repair attempt failed: ' . $e->getMessage());
711 - }
712 542
713 - // Method 4: Manual PDF structure analysis (basic page count)
714 - try {
715 - $page_count = $this->mxchat_manual_pdf_page_count($pdf_path);
716 - if ($page_count > 0) {
717 - //error_log('PDF page count determined manually: ' . $page_count . ' pages');
718 - return $page_count;
543 + // Initialize failed pages list if it doesn't exist
544 + if (!isset($status['failed_pages_list']) || !is_array($status['failed_pages_list'])) {
545 + $status['failed_pages_list'] = [];
719 546 }
720 - } catch (Exception $e) {
721 - //error_log('Manual PDF analysis failed: ' . $e->getMessage());
722 - }
723 547
724 - //error_log('All PDF parsing methods failed for: ' . $pdf_path);
725 - return false;
726 -}
727 -
728 -/**
729 - * Check if shell_exec is disabled
730 - */
731 -private function mxchat_is_shell_disabled() {
732 - $disabled = explode(',', ini_get('disable_functions'));
733 - return in_array('shell_exec', $disabled);
734 -}
735 -
736 -/**
737 - * Attempt to repair PDF using basic methods
738 - */
739 -private function mxchat_attempt_pdf_repair($pdf_path) {
740 - try {
741 - $content = file_get_contents($pdf_path);
742 - if (!$content) {
743 - return false;
548 + $start_page = absint($status['processed_pages']);
549 + $end_page = min($start_page + $batch_size, $total_pages);
550 + $options = get_option('mxchat_options');
551 +
552 + if (empty($options['api_key'])) {
553 + throw new Exception('API key is missing or invalid');
744 554 }
745 555
746 - // Check if PDF starts with proper header
747 - if (substr($content, 0, 4) !== '%PDF') {
748 - // Try to find PDF header in the content
749 - $header_pos = strpos($content, '%PDF');
750 - if ($header_pos !== false && $header_pos < 1024) {
751 - // Remove junk before PDF header
752 - $content = substr($content, $header_pos);
753 - $repaired_path = $pdf_path . '.repaired';
754 - file_put_contents($repaired_path, $content);
755 - return $repaired_path;
756 - }
757 - }
556 + $successful_pages = 0;
557 + $failed_pages = 0;
758 558
759 - // Check for EOF marker
760 - $content = rtrim($content);
761 - if (!preg_match('/%%EOF\s*$/', $content)) {
762 - // Add EOF marker if missing
763 - $content .= "\n%%EOF";
764 - $repaired_path = $pdf_path . '.repaired';
765 - file_put_contents($repaired_path, $content);
766 - return $repaired_path;
767 - }
559 + for ($i = $start_page; $i < $end_page; $i++) {
560 + $page_number = $i + 1;
561 + $max_retries = 3;
562 + $retry_count = 0;
563 + $page_processed = false;
564 + $last_error = '';
768 565
769 - } catch (Exception $e) {
770 - //error_log('PDF repair error: ' . $e->getMessage());
771 - }
566 + while (!$page_processed && $retry_count < $max_retries) {
567 + try {
568 + $text = $pages[$i]->getText();
569 +
570 + if (empty($text)) {
571 + throw new Exception("Empty text on page {$page_number}");
572 + }
573 +
574 + $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
772 575
773 - return false;
774 -}
576 + if (empty($sanitized_content)) {
577 + throw new Exception("No valid content after sanitization on page {$page_number}");
578 + }
775 579
776 -/**
777 - * Manual PDF page counting by analyzing PDF structure
778 - */
779 -private function mxchat_manual_pdf_page_count($pdf_path) {
780 - try {
781 - $content = file_get_contents($pdf_path);
782 - if (!$content) {
783 - return 0;
784 - }
580 + $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
581 +
582 + if (is_string($embedding_vector)) {
583 + throw new Exception("Embedding generation failed: " . $embedding_vector);
584 + }
585 +
586 + if (!is_array($embedding_vector)) {
587 + throw new Exception("Embedding generation returned unexpected result type: " . gettype($embedding_vector));
588 + }
589 +
590 + $metadata = array(
591 + 'document_type' => 'pdf',
592 + 'total_pages' => $total_pages,
593 + 'current_page' => $page_number,
594 + 'prev_page' => $i > 0 ? $i : null,
595 + 'next_page' => $i < ($total_pages - 1) ? $i + 2 : null,
596 + 'source_url' => $pdf_url
597 + );
785 598
786 - // Method 1: Count /Type /Page objects
787 - $page_count = preg_match_all('/\/Type\s*\/Page[^s]/', $content);
788 - if ($page_count > 0) {
789 - return $page_count;
790 - }
599 + $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
600 + $page_url = esc_url($pdf_url . "#page=" . $page_number);
791 601
792 - // Method 2: Look for /Count in pages object
793 - if (preg_match('/\/Type\s*\/Pages.*?\/Count\s+(\d+)/', $content, $matches)) {
794 - return intval($matches[1]);
795 - }
602 + $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $options['api_key']);
603 +
604 + if (is_wp_error($db_result)) {
605 + throw new Exception("Database submission failed: " . $db_result->get_error_message());
606 + }
796 607
797 - // Method 3: Count page references
798 - $page_count = preg_match_all('/\d+\s+0\s+obj\s*<<[^>]*\/Type\s*\/Page/', $content);
799 - if ($page_count > 0) {
800 - return $page_count;
801 - }
802 -
803 - } catch (Exception $e) {
804 - //error_log('Manual PDF analysis error: ' . $e->getMessage());
805 - }
806 -
807 - return 0;
808 -}
809 -
810 -
811 -public function mxchat_save_inline_prompt() {
812 - // DEBUG: Log what we're receiving
813 - //error_log('=== MXCHAT DEBUG ===');
814 - //error_log('POST data: ' . print_r($_POST, true));
815 - //error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
816 -
817 - // Check for nonce security
818 - check_ajax_referer('mxchat_save_inline_nonce', '_ajax_nonce');
819 -
820 - // If we get here, nonce passed
821 - //error_log('Nonce verification PASSED');
822 -
823 - // Verify permissions
824 - if (!current_user_can('manage_options')) {
825 - wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
826 - return;
827 - }
828 -
829 - global $wpdb;
830 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
831 -
832 - // Validate and sanitize input data
833 - $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
834 - $article_content = isset($_POST['article_content']) ? wp_kses_post(wp_unslash($_POST['article_content'])) : '';
835 - $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
836 -
837 - if ($prompt_id > 0 && !empty($article_content)) {
838 - // Re-generate the embedding vector for the updated content
839 - $embedding_vector = $this->mxchat_generate_embedding($article_content);
840 - if (is_array($embedding_vector)) {
841 - // Serialize the embedding vector before storing it
842 - $embedding_vector_serialized = serialize($embedding_vector);
843 - // Update the prompt in the database
844 - $updated = $wpdb->update(
845 - $table_name,
846 - array(
847 - 'article_content' => $article_content,
848 - 'embedding_vector' => $embedding_vector_serialized,
849 - 'source_url' => $article_url,
850 - ),
851 - array('id' => $prompt_id),
852 - array('%s', '%s', '%s'),
853 - array('%d')
854 - );
855 - if ($updated !== false) {
856 - wp_send_json_success();
857 - } else {
858 - MxChat_Admin::mxchat_log_debug('knowledge_error', 'Database update failed when saving knowledge entry');
859 - wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
608 + // Success!
609 + $page_processed = true;
610 + $successful_pages++;
611 +
612 + } catch (Exception $e) {
613 + $retry_count++;
614 + $last_error = $e->getMessage();
615 +
616 + //error_log("PDF page {$page_number} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
617 +
618 + if ($retry_count < $max_retries) {
619 + // Wait before retry (exponential backoff: 1s, 2s, 4s)
620 + sleep(pow(2, $retry_count - 1));
621 + }
622 + }
860 623 }
861 - } else {
862 - MxChat_Admin::mxchat_log_debug('embedding_error', 'Embedding generation failed for knowledge entry');
863 - wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
864 - }
865 - } else {
866 - wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
867 - }
868 -}
869 624
870 -
871 -/**
872 - * AJAX: Get full content for editing — reassembles chunks if needed.
873 - * Works for both WordPress DB and Pinecone entries.
874 - */
875 -public function ajax_mxchat_get_entry_content() {
876 - check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
877 -
878 - if ( ! current_user_can('manage_options') ) {
879 - wp_send_json_error( array( 'message' => 'Permission denied.' ) );
880 - }
881 -
882 - $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
883 - $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
884 - $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
885 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
886 -
887 - if ( $data_source === 'pinecone' ) {
888 - // Pinecone: fetch vectors by source_url, reassemble chunks
889 - $content = $this->get_pinecone_entry_content( $source_url, $entry_id, $bot_id );
890 - } else {
891 - // WordPress DB
892 - $content = $this->get_wordpress_entry_content( $source_url, $entry_id );
893 - }
894 -
895 - if ( is_wp_error( $content ) ) {
896 - wp_send_json_error( array( 'message' => $content->get_error_message() ) );
897 - }
898 -
899 - wp_send_json_success( $content );
900 -}
901 -
902 -/**
903 - * Get content from WordPress DB — reassembles chunks by source_url.
904 - */
905 -private function get_wordpress_entry_content( $source_url, $entry_id ) {
906 - global $wpdb;
907 - $table = $wpdb->prefix . 'mxchat_system_prompt_content';
908 -
909 - // If we have a source_url, check for chunks
910 - if ( ! empty( $source_url ) && strpos( $source_url, 'mxchat://' ) !== 0 ) {
911 - $rows = $wpdb->get_results( $wpdb->prepare(
912 - "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
913 - $source_url
914 - ) );
915 -
916 - if ( $rows && count( $rows ) > 1 ) {
917 - // Multiple rows = chunked. Reassemble.
918 - $chunks = array();
919 - foreach ( $rows as $row ) {
920 - $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
921 - $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
922 - $chunks[ $index ] = $parsed['text'];
625 + // If page still not processed after all retries, mark as failed
626 + if (!$page_processed) {
627 + $failed_pages++;
628 + $status['failed_pages_list'][] = [
629 + 'page' => $page_number,
630 + 'error' => $last_error,
631 + 'time' => time(),
632 + 'retries' => $max_retries
633 + ];
634 +
635 + // Limit failed pages list to prevent memory issues
636 + if (count($status['failed_pages_list']) > 50) {
637 + $status['failed_pages_list'] = array_slice($status['failed_pages_list'], -50);
638 + }
639 +
640 + //error_log("PDF page {$page_number} permanently failed after {$max_retries} attempts: " . $last_error);
923 641 }
924 - ksort( $chunks );
925 - return array(
926 - 'content' => implode( "\n\n", $chunks ),
927 - 'source_url' => $source_url,
928 - 'is_chunked' => true,
929 - 'chunk_count' => count( $chunks ),
930 - 'content_type' => $rows[0]->content_type,
931 - );
932 - } elseif ( $rows && count( $rows ) === 1 ) {
933 - $parsed = MxChat_Chunker::parse_stored_chunk( $rows[0]->article_content );
934 - return array(
935 - 'content' => $parsed['text'],
936 - 'source_url' => $source_url,
937 - 'entry_id' => $rows[0]->id,
938 - 'is_chunked' => false,
939 - 'content_type' => $rows[0]->content_type,
940 - );
941 - }
942 - }
943 642
944 - // Fallback: fetch by ID
945 - if ( $entry_id > 0 ) {
946 - $row = $wpdb->get_row( $wpdb->prepare(
947 - "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
948 - $entry_id
949 - ) );
950 - if ( $row ) {
951 - $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
952 - return array(
953 - 'content' => $parsed['text'],
954 - 'source_url' => $row->source_url,
955 - 'entry_id' => $row->id,
956 - 'is_chunked' => false,
957 - 'content_type' => $row->content_type,
958 - );
643 + // Update progress
644 + $status['processed_pages'] = absint($page_number);
645 + $status['last_update'] = time();
646 + $status['failed_pages'] = absint($status['failed_pages'] ?? 0) + ($page_processed ? 0 : 1);
647 +
648 + set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
959 649 }
960 - }
961 650
962 - return new WP_Error( 'not_found', 'Entry not found.' );
963 -}
964 -
965 -/**
966 - * Get content from Pinecone — fetches vectors by source_url, reassembles chunks.
967 - */
968 -private function get_pinecone_entry_content( $source_url, $entry_id, $bot_id ) {
969 - if ( ! class_exists('MxChat_Pinecone_Manager') ) {
970 - return new WP_Error( 'pinecone_unavailable', 'Pinecone manager not available.' );
971 - }
972 -
973 - // Get Pinecone config
974 - if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
975 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
976 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
977 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
978 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
979 - } else {
980 - $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
981 - $api_key = $bot_config['api_key'] ?? '';
982 - $host = $bot_config['host'] ?? '';
983 - $namespace = $bot_config['namespace'] ?? '';
984 - }
985 -
986 - if ( empty($host) || empty($api_key) ) {
987 - return new WP_Error( 'pinecone_config', 'Pinecone not configured.' );
988 - }
989 -
990 - // List vectors with the source_url prefix
991 - $base_id = md5( $source_url );
992 - $vector_ids = array( $base_id );
993 -
994 - // Find chunk vectors
995 - $list_url = "https://{$host}/vectors/list";
996 - $list_body = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
997 - if ( ! empty($namespace) ) {
998 - $list_body['namespace'] = $namespace;
999 - }
1000 -
1001 - $list_resp = wp_remote_post( $list_url, array(
1002 - 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1003 - 'body' => wp_json_encode( $list_body ),
1004 - 'timeout' => 15,
1005 - ) );
1006 -
1007 - if ( ! is_wp_error($list_resp) ) {
1008 - $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
1009 - if ( ! empty($list_data['vectors']) ) {
1010 - foreach ( $list_data['vectors'] as $v ) {
1011 - $vector_ids[] = $v['id'];
651 + // Schedule next batch if needed
652 + if ($end_page < $total_pages) {
653 + wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_pdf_pages', array(
654 + 'pdf_path' => $pdf_path,
655 + 'pdf_url' => $pdf_url,
656 + 'total_pages' => $total_pages,
657 + 'batch_size' => $batch_size,
658 + 'batch_pause' => $batch_pause
659 + ));
660 + } else {
661 + // Processing complete
662 + $status['status'] = 'complete';
663 + $status['processed_pages'] = $total_pages;
664 +
665 + // Add completion summary
666 + $status['completion_summary'] = [
667 + 'total_pages' => $total_pages,
668 + 'successful_pages' => $total_pages - absint($status['failed_pages'] ?? 0),
669 + 'failed_pages' => absint($status['failed_pages'] ?? 0),
670 + 'completion_time' => current_time('mysql')
671 + ];
672 +
673 + // Save the completed status (don't delete it - let user dismiss manually)
674 + set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
675 +
676 + // Clean up the temporary PDF file
677 + if (file_exists($pdf_path)) {
678 + wp_delete_file($pdf_path);
1012 679 }
680 +
681 + // DON'T delete the status transients here - let user dismiss manually
1013 682 }
1014 - }
1015 683
1016 - // Fetch vectors with metadata
1017 - $fetch_url = "https://{$host}/vectors/fetch";
1018 - $fetch_body = array( 'ids' => $vector_ids );
1019 - if ( ! empty($namespace) ) {
1020 - $fetch_body['namespace'] = $namespace;
1021 - }
1022 -
1023 - $fetch_resp = wp_remote_post( $fetch_url, array(
1024 - 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1025 - 'body' => wp_json_encode( $fetch_body ),
1026 - 'timeout' => 15,
1027 - ) );
1028 -
1029 - if ( is_wp_error($fetch_resp) ) {
1030 - return new WP_Error( 'pinecone_fetch', 'Failed to fetch from Pinecone.' );
1031 - }
1032 -
1033 - $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
1034 - $vectors = $fetch_data['vectors'] ?? array();
1035 -
1036 - if ( empty($vectors) ) {
1037 - return new WP_Error( 'not_found', 'Entry not found in Pinecone.' );
1038 - }
1039 -
1040 - // Reassemble chunks
1041 - $chunks = array();
1042 - $content_type = 'content';
1043 - foreach ( $vectors as $vid => $vector ) {
1044 - $meta = $vector['metadata'] ?? array();
1045 - $text = $meta['text'] ?? '';
1046 - $index = $meta['chunk_index'] ?? 0;
1047 - $content_type = $meta['type'] ?? 'content';
1048 - $chunks[ intval($index) ] = $text;
1049 - }
1050 - ksort( $chunks );
1051 -
1052 - return array(
1053 - 'content' => implode( "\n\n", $chunks ),
1054 - 'source_url' => $source_url,
1055 - 'is_chunked' => count($chunks) > 1,
1056 - 'chunk_count' => count($chunks),
1057 - 'content_type' => $content_type,
1058 - );
1059 -}
1060 -
1061 -/**
1062 - * AJAX: Inspect a knowledge entry — returns the per-chunk stored text + metadata
1063 - * WITHOUT collapsing it, so a site owner can see exactly what was indexed for an
1064 - * entry (plan-mxchat-20260628-d8cb4b). READ-ONLY: never re-embeds or mutates.
1065 - */
1066 -public function ajax_mxchat_inspect_entry() {
1067 - check_ajax_referer('mxchat_inspect_entry_nonce', 'nonce');
1068 -
1069 - if ( ! current_user_can('manage_options') ) {
1070 - wp_send_json_error( array( 'message' => esc_html__('Permission denied.', 'mxchat') ) );
1071 - }
1072 -
1073 - $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
1074 - $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
1075 - $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
1076 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1077 -
1078 - if ( $data_source === 'pinecone' ) {
1079 - $result = $this->inspect_pinecone_entry( $source_url, $entry_id, $bot_id );
1080 - } else {
1081 - $result = $this->inspect_wordpress_entry( $source_url, $entry_id );
1082 - }
1083 -
1084 - if ( is_wp_error( $result ) ) {
1085 - wp_send_json_error( array( 'message' => $result->get_error_message() ) );
1086 - }
1087 -
1088 - wp_send_json_success( $result );
1089 -}
1090 -
1091 -/**
1092 - * Read-only inspector for WordPress-DB entries. Mirrors get_wordpress_entry_content()
1093 - * but returns each STORED chunk's exact text + length (no implode), plus the assembled
1094 - * embedded text. This shows what is actually in the index, not a re-derivation from the post.
1095 - */
1096 -private function inspect_wordpress_entry( $source_url, $entry_id ) {
1097 - global $wpdb;
1098 - $table = $wpdb->prefix . 'mxchat_system_prompt_content';
1099 -
1100 - $rows = array();
1101 -
1102 - // Group by the real stored source_url — this INCLUDES "mxchat://" manual
1103 - // Direct Content entries (the spec's manual-entry case), which share one
1104 - // source_url across their chunk rows. Only the synthetic "_ungrouped_<id>"
1105 - // display key (invented by the table view for rows with no source_url) is
1106 - // excluded; those fall through to the entry_id lookup below.
1107 - if ( ! empty( $source_url ) && strpos( $source_url, '_ungrouped_' ) !== 0 ) {
1108 - $rows = $wpdb->get_results( $wpdb->prepare(
1109 - "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
1110 - $source_url
1111 - ) );
1112 - }
1113 -
1114 - // Fallback / manual "Direct Content" entries: fetch the single row by id.
1115 - if ( empty( $rows ) && $entry_id > 0 ) {
1116 - $row = $wpdb->get_row( $wpdb->prepare(
1117 - "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
1118 - $entry_id
1119 - ) );
1120 - if ( $row ) {
1121 - $rows = array( $row );
684 + } catch (\Exception $e) {
685 + //error_log(sprintf('[MXCHAT-PDF] Error processing PDF: %s', $e->getMessage()));
686 +
687 + $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
688 + $status = get_transient($status_key);
689 +
690 + if (!$status || !is_array($status)) {
691 + $status = array(
692 + 'total_pages' => $total_pages,
693 + 'processed_pages' => 0,
694 + 'status' => 'error',
695 + 'error' => sanitize_text_field($e->getMessage()),
696 + 'last_update' => time()
697 + );
698 + } else {
699 + $status['status'] = 'error';
700 + $status['error'] = sanitize_text_field($e->getMessage());
701 + $status['last_update'] = time();
1122 702 }
1123 - }
1124 -
1125 - if ( empty( $rows ) ) {
1126 - return new WP_Error( 'not_found', esc_html__('Entry not found in the local knowledge database.', 'mxchat') );
1127 - }
1128 -
1129 - $chunks = array();
1130 - $content_type = '';
1131 - foreach ( $rows as $row ) {
1132 - $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
1133 - $text = isset( $parsed['text'] ) ? $parsed['text'] : '';
1134 - $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
1135 - $content_type = $row->content_type;
1136 - $chunks[] = array(
1137 - 'index' => $index,
1138 - 'text' => $text,
1139 - 'length' => function_exists('mb_strlen') ? mb_strlen( $text ) : strlen( $text ),
1140 - 'row_id' => intval( $row->id ),
1141 - );
1142 - }
1143 -
1144 - usort( $chunks, function( $a, $b ) { return $a['index'] - $b['index']; } );
1145 -
1146 - $assembled = implode( "\n\n", wp_list_pluck( $chunks, 'text' ) );
1147 -
1148 - return array(
1149 - 'store' => 'wordpress',
1150 - 'source_url' => $source_url,
1151 - 'content_type' => $content_type,
1152 - 'is_chunked' => count( $chunks ) > 1,
1153 - 'chunk_count' => count( $chunks ),
1154 - 'assembled' => $assembled,
1155 - 'assembled_length' => function_exists('mb_strlen') ? mb_strlen( $assembled ) : strlen( $assembled ),
1156 - 'chunks' => array_values( $chunks ),
1157 - // WP-DB storage carries no separate vector metadata; surface that fact
1158 - // rather than letting the owner guess (the spec's taxonomy question).
1159 - 'metadata' => array(),
1160 - 'metadata_note' => esc_html__('Stored in the local WordPress database. Only the assembled text shown here is embedded — there are no separate vector metadata fields (e.g. taxonomy terms are not stored unless they were injected into the text itself).', 'mxchat'),
1161 - );
1162 -}
1163 -
1164 -/**
1165 - * Read-only inspector for Pinecone entries. Mirrors get_pinecone_entry_content()
1166 - * but keeps each vector's text + metadata instead of imploding, so the owner can
1167 - * confirm exactly which metadata fields (text/source_url/type/last_updated/created_at/bot_id)
1168 - * are present per chunk. READ-ONLY.
1169 - */
1170 -private function inspect_pinecone_entry( $source_url, $entry_id, $bot_id ) {
1171 - if ( ! class_exists('MxChat_Pinecone_Manager') ) {
1172 - return new WP_Error( 'pinecone_unavailable', esc_html__('Pinecone manager not available.', 'mxchat') );
1173 - }
1174 -
1175 - if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
1176 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
1177 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1178 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1179 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1180 - } else {
1181 - $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1182 - $api_key = $bot_config['api_key'] ?? '';
1183 - $host = $bot_config['host'] ?? '';
1184 - $namespace = $bot_config['namespace'] ?? '';
1185 - }
1186 -
1187 - if ( empty($host) || empty($api_key) ) {
1188 - return new WP_Error( 'pinecone_config', esc_html__('Pinecone not configured.', 'mxchat') );
1189 - }
1190 -
1191 - $base_id = md5( $source_url );
1192 - $vector_ids = array( $base_id );
1193 -
1194 - $list_url = "https://{$host}/vectors/list";
1195 - $list_body = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
1196 - if ( ! empty($namespace) ) {
1197 - $list_body['namespace'] = $namespace;
1198 - }
1199 -
1200 - $list_resp = wp_remote_post( $list_url, array(
1201 - 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1202 - 'body' => wp_json_encode( $list_body ),
1203 - 'timeout' => 15,
1204 - ) );
1205 -
1206 - if ( ! is_wp_error($list_resp) ) {
1207 - $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
1208 - if ( ! empty($list_data['vectors']) ) {
1209 - foreach ( $list_data['vectors'] as $v ) {
1210 - $vector_ids[] = $v['id'];
1211 - }
703 +
704 + set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
705 +
706 + if (file_exists($pdf_path)) {
707 + wp_delete_file($pdf_path);
1212 708 }
1213 709 }
1214 -
1215 - $fetch_url = "https://{$host}/vectors/fetch";
1216 - $fetch_body = array( 'ids' => $vector_ids );
1217 - if ( ! empty($namespace) ) {
1218 - $fetch_body['namespace'] = $namespace;
1219 - }
1220 -
1221 - $fetch_resp = wp_remote_post( $fetch_url, array(
1222 - 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1223 - 'body' => wp_json_encode( $fetch_body ),
1224 - 'timeout' => 15,
1225 - ) );
1226 -
1227 - if ( is_wp_error($fetch_resp) ) {
1228 - return new WP_Error( 'pinecone_fetch', esc_html__('Failed to fetch from Pinecone.', 'mxchat') );
1229 - }
1230 -
1231 - $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
1232 - $vectors = $fetch_data['vectors'] ?? array();
1233 -
1234 - if ( empty($vectors) ) {
1235 - return new WP_Error( 'not_found', esc_html__('Entry not found in Pinecone.', 'mxchat') );
1236 - }
1237 -
1238 - // Whitelisted metadata fields the spec calls out — shown so devs can confirm
1239 - // what is (and is NOT) stored per vector.
1240 - $meta_fields = array( 'text', 'source_url', 'type', 'last_updated', 'created_at', 'bot_id', 'chunk_index', 'total_chunks' );
1241 - $chunks = array();
1242 - $content_type = '';
1243 - foreach ( $vectors as $vid => $vector ) {
1244 - $meta = isset($vector['metadata']) && is_array($vector['metadata']) ? $vector['metadata'] : array();
1245 - $text = $meta['text'] ?? '';
1246 - $index = isset($meta['chunk_index']) ? intval($meta['chunk_index']) : count($chunks);
1247 - $content_type = $meta['type'] ?? $content_type;
1248 -
1249 - $clean_meta = array();
1250 - foreach ( $meta_fields as $field ) {
1251 - if ( array_key_exists( $field, $meta ) && $field !== 'text' ) {
1252 - $clean_meta[ $field ] = is_scalar( $meta[ $field ] ) ? (string) $meta[ $field ] : wp_json_encode( $meta[ $field ] );
1253 - }
1254 - }
1255 -
1256 - $chunks[] = array(
1257 - 'index' => $index,
1258 - 'text' => $text,
1259 - 'length' => function_exists('mb_strlen') ? mb_strlen( $text ) : strlen( $text ),
1260 - 'vector_id' => (string) $vid,
1261 - 'metadata' => $clean_meta,
1262 - );
1263 - }
1264 -
1265 - usort( $chunks, function( $a, $b ) { return $a['index'] - $b['index']; } );
1266 -
1267 - $assembled = implode( "\n\n", wp_list_pluck( $chunks, 'text' ) );
1268 -
1269 - return array(
1270 - 'store' => 'pinecone',
1271 - 'source_url' => $source_url,
1272 - 'content_type' => $content_type,
1273 - 'is_chunked' => count( $chunks ) > 1,
1274 - 'chunk_count' => count( $chunks ),
1275 - 'assembled' => $assembled,
1276 - 'assembled_length' => function_exists('mb_strlen') ? mb_strlen( $assembled ) : strlen( $assembled ),
1277 - 'chunks' => array_values( $chunks ),
1278 - 'metadata' => array(),
1279 - 'metadata_note' => esc_html__('Stored in Pinecone. Each chunk above lists the vector metadata fields actually present — if a field you expect (such as taxonomy terms) is missing here, it was not stored as metadata and is only searchable if it appears in the embedded text.', 'mxchat'),
1280 - );
1281 710 }
1282 711
1283 -/**
1284 - * AJAX: Save edited content — re-chunks and re-embeds as needed.
1285 - * Works for both WordPress DB and Pinecone entries.
1286 - */
1287 -public function ajax_mxchat_save_entry_content() {
1288 - check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
712 + public function mxchat_save_inline_prompt() {
713 + // Check for nonce security
714 + check_ajax_referer('mxchat_save_inline_nonce');
1289 715
1290 - if ( ! current_user_can('manage_options') ) {
1291 - wp_send_json_error( array( 'message' => 'Permission denied.' ) );
1292 - }
716 + // Verify permissions
717 + if (!current_user_can('manage_options')) {
718 + wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
719 + return;
720 + }
1293 721
1294 - $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
1295 - $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
1296 - $content = isset($_POST['content']) ? wp_kses_post( wp_unslash($_POST['content']) ) : '';
1297 - $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
1298 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1299 - $content_type = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : 'content';
722 + global $wpdb;
723 + $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1300 724
1301 - if ( empty($content) ) {
1302 - wp_send_json_error( array( 'message' => 'Content cannot be empty.' ) );
1303 - }
725 + // Validate and sanitize input data
726 + $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
727 + $article_content = isset($_POST['article_content']) ? sanitize_textarea_field($_POST['article_content']) : '';
728 + $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
1304 729
1305 - // Get the embedding API key
1306 - $options = get_option('mxchat_options', array());
1307 - $api_key = '';
730 + if ($prompt_id > 0 && !empty($article_content)) {
731 + // Re-generate the embedding vector for the updated content
732 + $embedding_vector = $this->mxchat_generate_embedding($article_content);
1308 733
1309 - if ( $bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager') ) {
1310 - $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
1311 - $api_key = $bot_options['api_key'] ?? '';
1312 - }
1313 - if ( empty($api_key) ) {
1314 - $api_key = $options['api_key'] ?? '';
1315 - }
734 + if (is_array($embedding_vector)) {
735 + // Serialize the embedding vector before storing it
736 + $embedding_vector_serialized = serialize($embedding_vector);
1316 737
1317 - global $wpdb;
1318 - $table = $wpdb->prefix . 'mxchat_system_prompt_content';
738 + // Update the prompt in the database
739 + $updated = $wpdb->update(
740 + $table_name,
741 + array(
742 + 'article_content' => $article_content,
743 + 'embedding_vector' => $embedding_vector_serialized,
744 + 'source_url' => $article_url,
745 + ),
746 + array('id' => $prompt_id),
747 + array('%s', '%s', '%s'),
748 + array('%d')
749 + );
1319 750
1320 - // If source_url is empty but we have an entry_id, look it up
1321 - if ( empty($source_url) && $entry_id > 0 && $data_source === 'wordpress' ) {
1322 - $row = $wpdb->get_row( $wpdb->prepare( "SELECT source_url FROM {$table} WHERE id = %d", $entry_id ) );
1323 - if ( $row && ! empty($row->source_url) ) {
1324 - $source_url = $row->source_url;
1325 - }
1326 - }
751 + if ($updated !== false) {
752 + wp_send_json_success();
753 + } else {
754 + wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
755 + }
756 + } else {
757 + wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
758 + }
759 + } else {
760 + wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
761 + }
762 + }
1327 763
1328 - // For manual entries (no source_url or mxchat:// prefix), delete the old entry by ID first
1329 - // so submit_content_to_db creates a replacement instead of a duplicate
1330 - // Also treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
1331 - $is_legacy_manual = !empty($source_url) && strpos($source_url, 'mxchat.ai') !== false && strpos($source_url, 'mxchat://') !== 0;
1332 - if ( $entry_id > 0 && (empty($source_url) || strpos($source_url, 'mxchat://') === 0 || $is_legacy_manual) ) {
1333 - $wpdb->delete( $table, array( 'id' => $entry_id ), array( '%d' ) );
1334 - // Clear legacy URL so submit_content_to_db generates a unique mxchat:// identifier
1335 - // instead of reusing the shared URL (which would mass-delete other entries with the same URL)
1336 - if ( $is_legacy_manual ) {
1337 - $source_url = '';
1338 - }
1339 - }
1340 764
1341 - // Use the existing submit_content_to_db which handles chunking, Pinecone, and WP DB
1342 - $vector_id = ! empty($source_url) ? md5($source_url) : md5('mxchat_manual_' . $entry_id);
1343 -
1344 - // submit_content_to_db already handles: delete old chunks → re-chunk → re-embed → store
1345 - $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, $vector_id, $bot_id, $content_type );
1346 -
1347 - if ( is_wp_error($result) ) {
1348 - wp_send_json_error( array( 'message' => $result->get_error_message() ) );
1349 - }
1350 -
1351 - wp_send_json_success( array( 'message' => 'Content saved and re-embedded successfully.' ) );
1352 -}
1353 -
1354 765 public function mxchat_get_pdf_processing_status($pdf_url) {
1355 766 $pdf_url = esc_url_raw($pdf_url);
1356 767 $status = get_transient(sanitize_key('mxchat_pdf_status_' . md5($pdf_url)));
1357 768
@@ -1394,18 +805,24 @@
1394 805 }
1395 806
1396 807
1397 808 public function mxchat_handle_sitemap_submission() {
809 + // Start logging the submission process
810 + //error_log('[MXCHAT-URL] ===== Starting URL submission process =====');
811 +
1398 812 // Check if the form was submitted and verify permissions
1399 813 if (!isset($_POST['submit_sitemap']) || !current_user_can('manage_options')) {
814 + //error_log('[MXCHAT-URL] Error: Unauthorized access or form not submitted properly');
1400 815 wp_die(esc_html__('Unauthorized access', 'mxchat'));
1401 816 }
1402 817
1403 818 // Verify nonce
819 + //error_log('[MXCHAT-URL] Verifying nonce');
1404 820 check_admin_referer('mxchat_submit_sitemap_action', 'mxchat_submit_sitemap_nonce');
1405 821
1406 822 // Validate URL
1407 823 if (!isset($_POST['sitemap_url']) || empty($_POST['sitemap_url'])) {
824 + //error_log('[MXCHAT-URL] Error: Empty or missing URL');
1408 825 set_transient('mxchat_admin_notice_error',
1409 826 esc_html__('Please provide a valid URL.', 'mxchat'),
1410 827 30
1411 828 );
@@ -1413,56 +830,43 @@
1413 830 exit;
1414 831 }
1415 832
1416 833 $submitted_url = esc_url_raw($_POST['sitemap_url']);
834 + //error_log('[MXCHAT-URL] Processing URL: ' . $submitted_url);
1417 835
1418 - // Convert Google Drive sharing URLs to direct download URLs
1419 - if ( strpos($submitted_url, 'drive.google.com') !== false ) {
1420 - $file_id = '';
1421 - if ( preg_match('/[?&]id=([a-zA-Z0-9_-]+)/', $submitted_url, $m) ) {
1422 - $file_id = $m[1];
1423 - } elseif ( preg_match('#/file/d/([a-zA-Z0-9_-]+)#', $submitted_url, $m) ) {
1424 - $file_id = $m[1];
1425 - }
1426 - if ( ! empty($file_id) ) {
1427 - $submitted_url = 'https://drive.google.com/uc?export=download&id=' . $file_id;
1428 - }
836 + // Validate API key first
837 + $options = get_option('mxchat_options');
838 + $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
839 +
840 + if (strpos($selected_model, 'voyage') === 0) {
841 + $api_key = $options['voyage_api_key'] ?? '';
842 + $provider_name = 'Voyage AI';
843 + } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
844 + $api_key = $options['gemini_api_key'] ?? '';
845 + $provider_name = 'Google Gemini';
846 + } else {
847 + $api_key = $options['api_key'] ?? '';
848 + $provider_name = 'OpenAI';
1429 849 }
1430 -
1431 - // Get bot_id from form submission
1432 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1433 -
1434 - // Get bot-specific options and validate the embedding decision —
1435 - // custom-provider-aware (plan cbd5fd).
1436 - $bot_options = $this->get_bot_options($bot_id);
1437 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
1438 -
1439 - $preflight = MxChat_Utils::embedding_preflight($options);
1440 - if (!$preflight['ok']) {
1441 - set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
850 +
851 + if (empty($api_key)) {
852 + $error_message = sprintf(
853 + esc_html__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
854 + $provider_name
855 + );
856 + //error_log('[MXCHAT-URL] Error: ' . $error_message);
857 + set_transient('mxchat_admin_notice_error', $error_message, 30);
858 + //error_log('[MXCHAT-URL] Set error transient: ' . $error_message);
1442 859 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1443 860 exit;
1444 861 }
1445 - $api_key = $preflight['api_key'];
1446 862
1447 - // Fetch URL — send an honest, versioned MXChat crawler UA (not a spoofed
1448 - // browser). Stale browser UAs are exactly what WAFs like SiteGround's
1449 - // ModSecurity flag as scrapers, 403-ing the fetch (including PDFs served
1450 - // from the site's own media library, which route through this same call).
1451 - // See mxchat_ingest_user_agent(). Accept is kept for content negotiation;
1452 - // the browser-only Accept-Language fingerprint is dropped so it stays
1453 - // coherent with a bot identity.
1454 - $response = wp_remote_get($submitted_url, array(
1455 - 'timeout' => 30,
1456 - 'sslverify' => false,
1457 - 'user-agent' => mxchat_ingest_user_agent(),
1458 - 'headers' => array(
1459 - 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1460 - ),
1461 - ));
863 + //error_log('[MXCHAT-URL] Fetching URL content');
864 + $response = wp_remote_get($submitted_url, array('timeout' => 30));
1462 865
1463 866 if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
1464 867 $error_message = is_wp_error($response) ? $response->get_error_message() : 'HTTP Status: ' . wp_remote_retrieve_response_code($response);
868 + //error_log('[MXCHAT-URL] Error fetching URL: ' . $error_message);
1465 869 set_transient('mxchat_admin_notice_error',
1466 870 sprintf(
1467 871 esc_html__('Failed to fetch the URL: %s', 'mxchat'),
1468 872 esc_html($error_message)
@@ -1473,11 +877,13 @@
1473 877 exit;
1474 878 }
1475 879
1476 880 $content_type = wp_remote_retrieve_header($response, 'content-type');
881 + //error_log('[MXCHAT-URL] Content type: ' . $content_type);
1477 882 $body_content = wp_remote_retrieve_body($response);
1478 883
1479 884 if (empty($body_content)) {
885 + //error_log('[MXCHAT-URL] Error: Empty response body');
1480 886 set_transient('mxchat_admin_notice_error',
1481 887 esc_html__('Empty response received from URL.', 'mxchat'),
1482 888 30
1483 889 );
@@ -1483,21 +889,29 @@
1483 889 );
1484 890 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1485 891 exit;
1486 892 }
893 + //error_log('[MXCHAT-URL] Retrieved body content length: ' . strlen($body_content) . ' bytes');
1487 894
1488 895 // Handle PDF URL
1489 896 if ($this->mxchat_is_pdf_url($submitted_url, $response)) {
1490 - $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response, $bot_id);
897 + //error_log('[MXCHAT-URL] Detected PDF URL, handling PDF for knowledge base');
898 + $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response);
899 + //error_log('[MXCHAT-URL] PDF handling result: ' . $result);
1491 900
1492 - if ($result === 'queued') {
1493 - set_transient('mxchat_admin_notice_success',
1494 - esc_html__('PDF queued for processing. Processing will start automatically.', 'mxchat'),
901 + if ($result === 'scheduled') {
902 + set_transient(
903 + 'mxchat_last_pdf_url',
904 + sanitize_text_field($submitted_url),
905 + DAY_IN_SECONDS
906 + );
907 + set_transient('mxchat_admin_notice_info',
908 + esc_html__('PDF processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1495 909 30
1496 910 );
1497 911 } else {
1498 912 set_transient('mxchat_admin_notice_error',
1499 - esc_html__('Failed to queue PDF processing: ', 'mxchat') . esc_html($result),
913 + esc_html__('Failed to start PDF processing: ', 'mxchat') . esc_html($result),
1500 914 30
1501 915 );
1502 916 }
1503 917
@@ -1506,8 +920,9 @@
1506 920 }
1507 921
1508 922 // Handle Sitemap XML
1509 923 if (strpos($content_type, 'xml') !== false || strpos($body_content, '<urlset') !== false) {
924 + //error_log('[MXCHAT-URL] Detected XML content, processing as sitemap');
1510 925 libxml_use_internal_errors(true);
1511 926 $xml = simplexml_load_string($body_content);
1512 927 $xml_errors = libxml_get_errors();
1513 928 libxml_clear_errors();
@@ -1512,8 +927,15 @@
1512 927 $xml_errors = libxml_get_errors();
1513 928 libxml_clear_errors();
1514 929
1515 930 if ($xml === false || !empty($xml_errors)) {
931 + //error_log('[MXCHAT-URL] Error: Invalid XML format');
932 + if (!empty($xml_errors)) {
933 + foreach ($xml_errors as $error) {
934 + //error_log('[MXCHAT-URL] XML Error: ' . $error->message);
935 + }
936 + }
937 +
1516 938 set_transient('mxchat_admin_notice_error',
1517 939 esc_html__('Invalid sitemap XML. Please provide a valid sitemap.', 'mxchat'),
1518 940 30
1519 941 );
@@ -1520,18 +942,27 @@
1520 942 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1521 943 exit;
1522 944 }
1523 945
1524 - $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url, $bot_id);
946 + //error_log('[MXCHAT-URL] Valid XML found, handling sitemap for knowledge base');
947 + $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url);
948 + //error_log('[MXCHAT-URL] Sitemap handling result: ' . $result);
1525 949
1526 - if ($result === 'queued') {
1527 - set_transient('mxchat_admin_notice_success',
1528 - esc_html__('Sitemap queued for processing. Processing will start automatically.', 'mxchat'),
950 + if ($result === 'scheduled') {
951 + set_transient(
952 + 'mxchat_last_sitemap_url',
953 + sanitize_text_field($submitted_url),
954 + DAY_IN_SECONDS
955 + );
956 + set_transient('mxchat_admin_notice_info',
957 + esc_html__('Sitemap processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1529 958 30
1530 959 );
1531 960 } else {
961 + // Return to the admin page without a redirect for better error display
962 + // The error is already stored in the sitemap status transient
1532 963 set_transient('mxchat_admin_notice_error',
1533 - esc_html__('Failed to queue sitemap processing. Please check the status below for details.', 'mxchat'),
964 + esc_html__('Failed to start sitemap processing. Please check the status below for details.', 'mxchat'),
1534 965 30
1535 966 );
1536 967 }
1537 968
@@ -1538,50 +969,127 @@
1538 969 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1539 970 exit;
1540 971 }
1541 972
1542 - // Handle Regular URL (single page)
973 + // Handle Regular URL
974 + //error_log('[MXCHAT-URL] Processing as regular webpage');
1543 975 $page_content = $this->mxchat_extract_main_content($body_content);
976 + //error_log('[MXCHAT-URL] Extracted content length: ' . strlen($page_content) . ' bytes');
977 +
1544 978 $sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
979 + //error_log('[MXCHAT-URL] Sanitized content length: ' . strlen($sanitized_content) . ' bytes');
1545 980
1546 - //error_log('[MXCHAT-URL-DEBUG] URL: ' . $submitted_url);
1547 - //error_log('[MXCHAT-URL-DEBUG] Extracted page_content length: ' . strlen($page_content));
1548 - //error_log('[MXCHAT-URL-DEBUG] Sanitized content length: ' . strlen($sanitized_content));
1549 - //error_log('[MXCHAT-URL-DEBUG] Content preview: ' . substr($sanitized_content, 0, 500));
1550 -
1551 981 if (empty($sanitized_content)) {
982 + //error_log('[MXCHAT-URL] Error: No valid content after sanitization');
983 +
984 + // Set both transients - the error notice and the URL status
1552 985 set_transient('mxchat_admin_notice_error',
1553 986 esc_html__('No valid content found on the provided URL.', 'mxchat'),
1554 987 30
1555 988 );
989 +
990 + // Set URL status transient
991 + set_transient('mxchat_single_url_status', [
992 + 'url' => $submitted_url,
993 + 'timestamp' => current_time('mysql'),
994 + 'status' => 'failed',
995 + 'error' => esc_html__('No valid content found on the provided URL.', 'mxchat')
996 + ], DAY_IN_SECONDS);
997 +
1556 998 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1557 999 exit;
1558 1000 }
1559 1001
1560 - // For single URLs, process immediately using submit_content_to_db
1561 - // This handles chunking automatically for large content
1562 - $db_result = MxChat_Utils::submit_content_to_db(
1563 - $sanitized_content,
1564 - $submitted_url,
1565 - $api_key,
1566 - null,
1567 - $bot_id,
1568 - 'url' // content_type
1569 - );
1002 + //error_log('[MXCHAT-URL] Generating embedding for content');
1003 + $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
1570 1004
1571 - if (is_wp_error($db_result)) {
1572 - $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
1005 + // Check if embedding_vector is a string (error message)
1006 + if (is_string($embedding_vector)) {
1007 + //error_log('[MXCHAT-URL] Error generating embedding: ' . $embedding_vector);
1008 + $error_message = esc_html__('Failed to generate embedding: ', 'mxchat') . esc_html($embedding_vector);
1009 + //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
1010 +
1011 + // Set both transients
1573 1012 set_transient('mxchat_admin_notice_error', $error_message, 30);
1574 - } else {
1013 +
1014 + // Set URL status transient
1015 + set_transient('mxchat_single_url_status', [
1016 + 'url' => $submitted_url,
1017 + 'timestamp' => current_time('mysql'),
1018 + 'status' => 'failed',
1019 + 'error' => $error_message
1020 + ], DAY_IN_SECONDS);
1021 +
1022 + wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1023 + exit;
1024 + }
1025 +
1026 + if (is_array($embedding_vector)) {
1027 + //error_log('[MXCHAT-URL] Successfully generated embedding with ' . count($embedding_vector) . ' dimensions');
1028 +
1029 + $db_result = MxChat_Utils::submit_content_to_db(
1030 + $sanitized_content,
1031 + $submitted_url,
1032 + $api_key
1033 + );
1034 +
1035 + if (is_wp_error($db_result)) {
1036 + //error_log('[MXCHAT-URL] Error: Failed to store content in database: ' . $db_result->get_error_message());
1037 + $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
1038 + //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
1039 +
1040 + // Set both transients
1041 + set_transient('mxchat_admin_notice_error', $error_message, 30);
1042 +
1043 + // Set URL status transient
1044 + set_transient('mxchat_single_url_status', [
1045 + 'url' => $submitted_url,
1046 + 'timestamp' => current_time('mysql'),
1047 + 'status' => 'failed',
1048 + 'error' => $error_message
1049 + ], DAY_IN_SECONDS);
1050 +
1051 + wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1052 + exit;
1053 + }
1054 +
1055 + //error_log('[MXCHAT-URL] Successfully stored content in database');
1575 1056 $success_message = esc_html__('URL content successfully submitted!', 'mxchat');
1057 + //error_log('[MXCHAT-URL] Setting success transient: ' . $success_message);
1058 +
1059 + // Set both transients
1576 1060 set_transient('mxchat_admin_notice_success', $success_message, 30);
1061 +
1062 + // Set URL status transient with success
1063 + set_transient('mxchat_single_url_status', [
1064 + 'url' => $submitted_url,
1065 + 'timestamp' => current_time('mysql'),
1066 + 'status' => 'complete',
1067 + 'content_length' => strlen($sanitized_content),
1068 + 'embedding_dimensions' => count($embedding_vector)
1069 + ], DAY_IN_SECONDS);
1070 +
1071 + } else {
1072 + //error_log('[MXCHAT-URL] Error: Failed to generate embedding. Unexpected result type: ' . gettype($embedding_vector));
1073 + $error_message = esc_html__('Failed to generate embedding: Unexpected result type. Please check your API key and try again.', 'mxchat');
1074 + //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
1075 +
1076 + // Set both transients
1077 + set_transient('mxchat_admin_notice_error', $error_message, 30);
1078 +
1079 + // Set URL status transient
1080 + set_transient('mxchat_single_url_status', [
1081 + 'url' => $submitted_url,
1082 + 'timestamp' => current_time('mysql'),
1083 + 'status' => 'failed',
1084 + 'error' => $error_message
1085 + ], DAY_IN_SECONDS);
1577 1086 }
1578 1087
1088 + //error_log('[MXCHAT-URL] ===== Completed URL submission process =====');
1579 1089 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1580 1090 exit;
1581 1091 }
1582 -
1583 -
1584 1092 public function mxchat_get_single_url_status() {
1585 1093 $status = get_transient('mxchat_single_url_status');
1586 1094 if (!$status) {
1587 1095 return null;
@@ -1593,11 +1101,13 @@
1593 1101 }
1594 1102
1595 1103 return $status;
1596 1104 }
1597 -
1598 -public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url, $bot_id = 'default') {
1105 +public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url) {
1106 + // Clear any single URL status when starting sitemap processing
1107 + delete_transient('mxchat_single_url_status');
1599 1108 if (!current_user_can('manage_options')) {
1109 + //error_log(esc_html__('Unauthorized sitemap processing attempt', 'mxchat'));
1600 1110 return false;
1601 1111 }
1602 1112
1603 1113 try {
@@ -1606,96 +1116,331 @@
1606 1116 if (!$xml || !is_object($xml)) {
1607 1117 throw new Exception(__('Invalid XML object provided', 'mxchat'));
1608 1118 }
1609 1119
1610 - // Get bot-specific embedding API for validation
1611 - $bot_options = $this->get_bot_options($bot_id);
1612 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
1120 + // Add embedding validation before processing
1121 + // Test embedding with a small sample text to verify API key is working
1122 + $test_result = $this->mxchat_generate_embedding("This is a test to verify the embedding API key is working.");
1613 1123
1614 - // Test the embedding API before processing
1615 - $test_phrase = "Test embedding generation for MxChat";
1616 - $test_result = $this->mxchat_generate_embedding($test_phrase, $bot_id);
1617 -
1124 + // Check if test_result is a string (error message) rather than an array (valid embedding)
1618 1125 if (is_string($test_result)) {
1126 + //error_log('[MXCHAT-URL] Embedding API validation failed: ' . $test_result);
1127 +
1128 + // Store the error in the status transient so it can be displayed later
1129 + $status_data = array(
1130 + 'total_urls' => 0,
1131 + 'processed_urls' => 0,
1132 + 'status' => 'error',
1133 + 'error' => __('Embedding API validation failed: ', 'mxchat') . $test_result,
1134 + 'last_update' => time()
1135 + );
1136 +
1137 + set_transient(
1138 + sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
1139 + array_map('sanitize_text_field', $status_data),
1140 + DAY_IN_SECONDS
1141 + );
1142 +
1619 1143 throw new Exception(__('Embedding API validation failed: ', 'mxchat') . $test_result);
1620 1144 }
1621 1145
1146 + // Make sure it's an array (valid embedding)
1622 1147 if (!is_array($test_result)) {
1148 + //error_log('[MXCHAT-URL] Embedding API returned unexpected result type: ' . gettype($test_result));
1623 1149 throw new Exception(__('Embedding API returned unexpected result type. Please check your configuration.', 'mxchat'));
1624 1150 }
1625 1151
1626 - // Extract URLs from sitemap
1627 - $urls = array();
1152 + $urls = [];
1628 1153 foreach ($xml->url as $url_element) {
1629 1154 $url = esc_url_raw((string)$url_element->loc);
1630 1155 if ($url) {
1631 - $urls[] = array('url' => $url);
1156 + $urls[] = $url;
1632 1157 }
1633 1158 }
1634 1159
1635 - $total_urls = count($urls);
1160 + $total_urls = absint(count($urls));
1636 1161
1637 1162 if ($total_urls < 1) {
1638 1163 throw new Exception(__('No valid URLs found in sitemap', 'mxchat'));
1639 1164 }
1640 1165
1641 - // Create unique queue ID
1642 - $queue_id = 'sitemap_' . md5($sitemap_url . time());
1166 + wp_schedule_single_event(time(), 'mxchat_process_sitemap_urls', array(
1167 + 'urls' => $urls,
1168 + 'sitemap_url' => $sitemap_url,
1169 + 'total_urls' => $total_urls,
1170 + 'batch_size' => absint(10),
1171 + 'batch_pause' => absint(5)
1172 + ));
1643 1173
1644 - // Add URLs to queue
1645 - $queued_count = $this->mxchat_add_to_queue($queue_id, 'url', $urls, $bot_id);
1174 + $status_data = array(
1175 + 'total_urls' => $total_urls,
1176 + 'processed_urls' => 0,
1177 + 'status' => 'processing',
1178 + 'last_update' => time()
1179 + );
1646 1180
1647 - if ($queued_count === 0) {
1648 - throw new Exception(__('Failed to add URLs to processing queue', 'mxchat'));
1649 - }
1181 + set_transient(
1182 + sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
1183 + array_map('sanitize_text_field', $status_data),
1184 + DAY_IN_SECONDS
1185 + );
1650 1186
1651 - // Store queue metadata
1652 - $this->mxchat_set_queue_meta($queue_id, 'source_url', $sitemap_url);
1653 - $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'sitemap');
1654 - $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_urls);
1655 - $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
1656 - $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
1187 + return __('scheduled', 'mxchat');
1657 1188
1658 - // Store queue ID in transient for status tracking
1659 - set_transient('mxchat_active_queue_sitemap', $queue_id, DAY_IN_SECONDS);
1660 - set_transient('mxchat_last_sitemap_url', $sitemap_url, DAY_IN_SECONDS);
1661 -
1662 - return 'queued';
1663 -
1664 - } catch (Exception $e) {
1189 + } catch (\Exception $e) {
1665 1190 $error_message = $e->getMessage();
1666 1191 //error_log(sprintf(esc_html__('Error preparing sitemap for processing: %s', 'mxchat'), esc_html($error_message)));
1667 1192
1193 + // Store the sitemap URL and error in transients so they can be displayed
1194 + set_transient(
1195 + 'mxchat_last_sitemap_url',
1196 + sanitize_text_field($sitemap_url),
1197 + DAY_IN_SECONDS
1198 + );
1199 +
1200 + $status_data = array(
1201 + 'total_urls' => 0,
1202 + 'processed_urls' => 0,
1203 + 'status' => 'error',
1204 + 'error' => $error_message,
1205 + 'last_update' => time()
1206 + );
1207 +
1208 + set_transient(
1209 + sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
1210 + array_map('sanitize_text_field', $status_data),
1211 + DAY_IN_SECONDS
1212 + );
1213 +
1668 1214 return $error_message;
1669 1215 }
1670 -
1671 1216 }
1672 1217
1673 -/**
1674 - * Remove shortcode tags but preserve the content inside them
1675 - * Example: [vc_column]Hello World[/vc_column] becomes "Hello World"
1676 - *
1677 - * @param string $content The content containing shortcodes
1678 - * @return string Content with shortcode tags removed but inner content preserved
1679 - */
1680 -private function strip_shortcode_tags_preserve_content($content) {
1681 - // Single-pass regex removes all shortcode brackets: [tag], [tag attr="val"], [/tag], [tag /]
1682 - // Content between tags is inherently preserved since only brackets are targeted
1683 - $result = preg_replace('/\[\/?\w[\w-]*[^\]]*\]/', '', $content);
1684 - return ($result !== null) ? $result : $content;
1218 +public function mxchat_process_sitemap_urls_cron($urls, $sitemap_url, $total_urls, $batch_size, $batch_pause) {
1219 + // Validate inputs
1220 + $sitemap_url = esc_url_raw($sitemap_url);
1221 + $total_urls = absint($total_urls);
1222 + $batch_size = absint($batch_size);
1223 + $batch_pause = absint($batch_pause);
1224 +
1225 + if (!is_array($urls) || empty($urls)) {
1226 + return;
1227 + }
1228 +
1229 + try {
1230 + $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
1231 + $status = get_transient($status_key);
1232 +
1233 + if (!$status || !is_array($status)) {
1234 + throw new Exception('Invalid status data retrieved from transient');
1235 + }
1236 +
1237 + // Initialize failed_urls array if it doesn't exist
1238 + if (!isset($status['failed_urls_list']) || !is_array($status['failed_urls_list'])) {
1239 + $status['failed_urls_list'] = [];
1240 + }
1241 +
1242 + $start_url = absint($status['processed_urls']);
1243 + $end_url = min($start_url + $batch_size, $total_urls);
1244 +
1245 + // Track batch statistics
1246 + $batch_stats = [
1247 + 'processed' => 0,
1248 + 'failed' => 0,
1249 + 'last_error' => '',
1250 + 'embedding_errors' => 0
1251 + ];
1252 +
1253 + // Check embedding configuration with first URL (only on first batch)
1254 + if ($start_url === 0) {
1255 + $test_url = esc_url_raw($urls[0]);
1256 + $test_response = wp_remote_get($test_url);
1257 +
1258 + if (!is_wp_error($test_response) && wp_remote_retrieve_response_code($test_response) === 200) {
1259 + $test_html = wp_remote_retrieve_body($test_response);
1260 + $test_content = $this->mxchat_extract_main_content($test_html);
1261 + $test_sanitized = $this->mxchat_sanitize_content_for_api($test_content);
1262 +
1263 + if (!empty($test_sanitized)) {
1264 + $test_embedding = $this->mxchat_generate_embedding($test_sanitized);
1265 +
1266 + if (is_string($test_embedding)) {
1267 + throw new Exception('Embedding generation failed: ' . $test_embedding);
1268 + }
1269 +
1270 + if (!is_array($test_embedding)) {
1271 + throw new Exception('Embedding generation returned unexpected result type: ' . gettype($test_embedding));
1272 + }
1273 + }
1274 + }
1275 + }
1276 +
1277 + for ($i = $start_url; $i < $end_url; $i++) {
1278 + $page_url = esc_url_raw($urls[$i]);
1279 + $max_retries = 3;
1280 + $retry_count = 0;
1281 + $url_processed = false;
1282 + $last_error = '';
1283 +
1284 + while (!$url_processed && $retry_count < $max_retries) {
1285 + try {
1286 + // Attempt to fetch the URL
1287 + $page_response = wp_remote_get($page_url, array('timeout' => 30));
1288 +
1289 + if (is_wp_error($page_response)) {
1290 + throw new Exception('HTTP request failed: ' . $page_response->get_error_message());
1291 + }
1292 +
1293 + $response_code = wp_remote_retrieve_response_code($page_response);
1294 + if ($response_code !== 200) {
1295 + throw new Exception('HTTP Status: ' . $response_code);
1296 + }
1297 +
1298 + $page_html = wp_remote_retrieve_body($page_response);
1299 +
1300 + if (empty($page_html)) {
1301 + throw new Exception('Empty response body');
1302 + }
1303 +
1304 + $page_content = $this->mxchat_extract_main_content($page_html);
1305 + $sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
1306 +
1307 + if (empty($sanitized_content)) {
1308 + throw new Exception('No valid content found after processing');
1309 + }
1310 +
1311 + $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
1312 +
1313 + if (is_string($embedding_vector)) {
1314 + throw new Exception('Embedding generation failed: ' . $embedding_vector);
1315 + }
1316 +
1317 + if (!is_array($embedding_vector)) {
1318 + throw new Exception('Embedding generation returned unexpected result type: ' . gettype($embedding_vector));
1319 + }
1320 +
1321 + // Submit to database
1322 + $options = get_option('mxchat_options');
1323 + $submission_result = MxChat_Utils::submit_content_to_db($sanitized_content, $page_url, $options['api_key']);
1324 +
1325 + if (is_wp_error($submission_result)) {
1326 + throw new Exception('Database submission failed: ' . $submission_result->get_error_message());
1327 + }
1328 +
1329 + // Success!
1330 + $url_processed = true;
1331 + $batch_stats['processed']++;
1332 +
1333 + } catch (Exception $e) {
1334 + $retry_count++;
1335 + $last_error = $e->getMessage();
1336 +
1337 + //error_log("URL {$page_url} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
1338 +
1339 + // Track embedding errors specifically
1340 + if (strpos($last_error, 'Embedding') !== false) {
1341 + $batch_stats['embedding_errors']++;
1342 + }
1343 +
1344 + if ($retry_count < $max_retries) {
1345 + // Wait before retry (exponential backoff: 1s, 2s, 4s)
1346 + sleep(pow(2, $retry_count - 1));
1347 + }
1348 + }
1349 + }
1350 +
1351 + // If URL still not processed after all retries, mark as failed
1352 + if (!$url_processed) {
1353 + $batch_stats['failed']++;
1354 + $batch_stats['last_error'] = $last_error;
1355 +
1356 + // Add to failed URLs list
1357 + $status['failed_urls_list'][] = [
1358 + 'url' => $page_url,
1359 + 'error' => $last_error,
1360 + 'time' => time(),
1361 + 'retries' => $max_retries
1362 + ];
1363 +
1364 + // Limit the number of failed URLs we store to prevent transient size issues
1365 + if (count($status['failed_urls_list']) > 100) {
1366 + $status['failed_urls_list'] = array_slice($status['failed_urls_list'], -100);
1367 + }
1368 + }
1369 +
1370 + // Update progress
1371 + $status['processed_urls'] = absint($i + 1);
1372 + $status['last_update'] = time();
1373 + $status['failed_urls'] = absint($status['failed_urls'] ?? 0) + ($url_processed ? 0 : 1);
1374 + $status['last_error'] = $batch_stats['last_error'];
1375 +
1376 + set_transient($status_key, $status, DAY_IN_SECONDS);
1377 +
1378 + // If we have too many consecutive embedding errors, stop processing
1379 + if ($batch_stats['embedding_errors'] >= 10) {
1380 + throw new Exception('Too many consecutive embedding failures detected. Please check your API configuration.');
1381 + }
1382 + }
1383 +
1384 + // If all URLs in this batch failed, stop processing
1385 + if ($batch_stats['processed'] === 0 && $batch_stats['failed'] > 0) {
1386 + $status['status'] = 'error';
1387 + $status['error'] = sprintf(
1388 + 'Processing stopped: %d consecutive failures in batch. Last error: %s',
1389 + $batch_stats['failed'],
1390 + $batch_stats['last_error']
1391 + );
1392 + set_transient($status_key, $status, DAY_IN_SECONDS);
1393 + return;
1394 + }
1395 +
1396 + // Update final progress
1397 + $status['processed_urls'] = min($end_url, $total_urls);
1398 + $status['last_update'] = time();
1399 + set_transient($status_key, $status, DAY_IN_SECONDS);
1400 +
1401 + // Check if we've processed all URLs
1402 + if ($end_url >= $total_urls) {
1403 + // All URLs have been processed - mark as complete
1404 + $status['status'] = 'complete';
1405 + $status['processed_urls'] = $total_urls;
1406 +
1407 + // Add completion summary
1408 + $status['completion_summary'] = [
1409 + 'total_urls' => $total_urls,
1410 + 'successful_urls' => $total_urls - absint($status['failed_urls'] ?? 0),
1411 + 'failed_urls' => absint($status['failed_urls'] ?? 0),
1412 + 'completion_time' => current_time('mysql')
1413 + ];
1414 +
1415 + // Save the completed status (don't delete it - let user dismiss manually)
1416 + set_transient($status_key, $status, DAY_IN_SECONDS);
1417 +
1418 + // DON'T delete the status transients here - let user dismiss manually
1419 + } else {
1420 + // Schedule next batch
1421 + wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_sitemap_urls', array(
1422 + 'urls' => $urls,
1423 + 'sitemap_url' => $sitemap_url,
1424 + 'total_urls' => $total_urls,
1425 + 'batch_size' => $batch_size,
1426 + 'batch_pause' => $batch_pause,
1427 + ));
1428 + }
1429 + } catch (\Exception $e) {
1430 + $status['status'] = 'error';
1431 + $status['error'] = $e->getMessage();
1432 + set_transient($status_key, $status, DAY_IN_SECONDS);
1433 + }
1685 1434 }
1686 -
1687 1435 public function mxchat_sanitize_content_for_api($content) {
1688 1436 //error_log('[MXCHAT-SANITIZE] Original content preview: ' . substr($content, 0, 500) . '...');
1689 -
1690 - // Remove shortcode tags but PRESERVE content inside them
1691 - $content = $this->strip_shortcode_tags_preserve_content($content);
1692 -
1437 +
1693 1438 // Remove script, style tags, and HTML comments
1694 1439 $content = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $content);
1695 1440 $content = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $content);
1696 1441 $content = preg_replace('/<!--(.|\s)*?-->/', '', $content);
1697 -
1442 +
1698 1443 // Remove all HTML tags and decode HTML entities
1699 1444 $content = wp_strip_all_tags($content);
1700 1445 $content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5);
1701 1446
@@ -1721,20 +1466,14 @@
1721 1466
1722 1467 // Ensure valid UTF-8 encoding
1723 1468 $content = wp_check_invalid_utf8($content);
1724 1469
1725 - // Remove extremely long runs with no whitespace (base64 blobs, minified JS).
1726 - // Counts CHARACTERS (/u), and never strips a run containing characters from a
1727 - // script written without spaces — Japanese, Chinese, Thai, Khmer, Lao, Myanmar —
1728 - // where a normal paragraph is legitimately one unbroken run.
1729 - $content = preg_replace_callback('/\S{300,}/u', function ($m) {
1730 - return preg_match('/[\p{Han}\p{Hiragana}\p{Katakana}\p{Thai}\p{Khmer}\p{Lao}\p{Myanmar}]/u', $m[0]) ? $m[0] : ' ';
1731 - }, $content);
1732 -
1733 - // Remove emoji/symbol blocks only — not the whole supplementary plane, which
1734 - // also holds CJK Extension B ideographs used in real Chinese/Japanese names
1735 - $content = preg_replace('/[\x{1F000}-\x{1F0FF}\x{1F300}-\x{1FAFF}]/u', '', $content);
1470 + // Remove any extremely long strings without spaces (often garbage)
1471 + $content = preg_replace('/\S{300,}/', ' ', $content);
1736 1472
1473 + // Replace problematic characters that often cause database issues
1474 + $content = preg_replace('/[\x{10000}-\x{10FFFF}]/u', '', $content); // Remove emoji and other high Unicode characters
1475 +
1737 1476 // Replace any remaining potentially problematic characters with spaces
1738 1477 // BUT preserve newlines by temporarily replacing them
1739 1478 $content = str_replace("\n", "NEWLINE_PLACEHOLDER", $content);
1740 1479 $content = preg_replace('/[^\p{L}\p{N}\p{P}\p{Z}\p{Sm}]/u', ' ', $content);
@@ -1739,13 +1478,12 @@
1739 1478 $content = str_replace("\n", "NEWLINE_PLACEHOLDER", $content);
1740 1479 $content = preg_replace('/[^\p{L}\p{N}\p{P}\p{Z}\p{Sm}]/u', ' ', $content);
1741 1480 $content = str_replace("NEWLINE_PLACEHOLDER", "\n", $content);
1742 1481
1743 - // Limit to reasonable length if needed (byte limit — MySQL TEXT is byte-sized,
1744 - // but cut on a character boundary so a multibyte char is never split mid-sequence)
1482 + // Limit to reasonable length if needed
1745 1483 $max_length = 65000; // Just under MySQL TEXT field limit
1746 1484 if (strlen($content) > $max_length) {
1747 - $content = mb_strcut($content, 0, $max_length, 'UTF-8');
1485 + $content = substr($content, 0, $max_length);
1748 1486 }
1749 1487
1750 1488 //error_log('[MXCHAT-SANITIZE] Sanitized content preview: ' . substr($content, 0, 500) . '...');
1751 1489 return $content;
@@ -1760,12 +1498,12 @@
1760 1498 @$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'), LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
1761 1499 $xpath = new DOMXPath($dom);
1762 1500
1763 1501 // For debugging purposes
1764 - $debugEnabled = true; // Set to true to enable debugging output
1502 + $debugEnabled = false; // Set to true to enable debugging output
1765 1503 $debug = function($message) use ($debugEnabled) {
1766 1504 if ($debugEnabled) {
1767 - //error_log('[MXCHAT-EXTRACT-DEBUG] ' . $message);
1505 + //error_log('[MXCHAT-DEBUG] ' . $message);
1768 1506 }
1769 1507 };
1770 1508
1771 1509 // Direct targeting for Gerow theme posts
@@ -1864,25 +1602,18 @@
1864 1602 '//main',
1865 1603 '//div[contains(@class, "content")]'
1866 1604 ];
1867 1605
1868 - // First handle Elementor content - get only leaf widget containers to avoid duplicates
1606 + // First handle Elementor content
1869 1607 $debug("Checking for Elementor content");
1870 - // Get widget containers that are direct children of widgets (not nested inside other widget containers)
1871 - $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-widget")]//div[contains(@class, "elementor-widget-container")]');
1608 + $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-element")]//div[contains(@class, "elementor-widget-container")]');
1872 1609 if ($elementor_widgets && $elementor_widgets->length > 0) {
1873 1610 $debug("Found Elementor widgets");
1874 - $seen_content = array(); // Track seen content to avoid duplicates
1875 1611 $combined_content = '';
1876 1612 foreach ($elementor_widgets as $widget) {
1877 1613 $widget_content = $dom->saveHTML($widget);
1878 1614 if (!empty($widget_content)) {
1879 - // Create a hash of the content to detect duplicates
1880 - $content_hash = md5($widget_content);
1881 - if (!isset($seen_content[$content_hash])) {
1882 - $seen_content[$content_hash] = true;
1883 - $combined_content .= $widget_content;
1884 - }
1615 + $combined_content .= $widget_content;
1885 1616 }
1886 1617 }
1887 1618 if (!empty($combined_content)) {
1888 1619 $debug("Returning Elementor content");
@@ -1894,14 +1625,15 @@
1894 1625 foreach ($selectors as $selector) {
1895 1626 $debug("Trying selector: " . $selector);
1896 1627 $nodes = $xpath->query($selector);
1897 1628 if ($nodes && $nodes->length > 0) {
1898 - $debug("Found " . $nodes->length . " matches for selector: " . $selector);
1899 - // Only take the FIRST matching node to avoid duplicate content
1900 - // (pages often have nested or multiple containers with same class)
1901 - $content = $dom->saveHTML($nodes->item(0));
1629 + $debug("Found matches for selector: " . $selector);
1630 + $content = '';
1631 + foreach ($nodes as $node) {
1632 + $content .= $dom->saveHTML($node);
1633 + }
1902 1634 if (!empty($content)) {
1903 - $debug("Returning content from selector: " . $selector . " (first match only)");
1635 + $debug("Returning content from selector: " . $selector);
1904 1636 return $content;
1905 1637 }
1906 1638 }
1907 1639 }
@@ -1925,106 +1657,16 @@
1925 1657 $debug("Returning blog-area section content");
1926 1658 return $content;
1927 1659 }
1928 1660 }
1929 -
1930 - // Generic container selectors for non-CMS sites (like .asp pages)
1931 - $debug("Trying generic container selectors");
1932 - $generic_selectors = [
1933 - '//div[@id="main"]',
1934 - '//div[@id="wrapper"]',
1935 - '//div[@id="page"]',
1936 - '//div[@id="site-content"]',
1937 - '//div[contains(@class, "main-content")]',
1938 - '//div[contains(@class, "page-content")]',
1939 - '//div[contains(@class, "site-content")]',
1940 - ];
1941 -
1942 - foreach ($generic_selectors as $selector) {
1943 - $debug("Trying generic selector: " . $selector);
1944 - $nodes = $xpath->query($selector);
1945 - if ($nodes && $nodes->length > 0) {
1946 - $content = $dom->saveHTML($nodes->item(0));
1947 - if (!empty($content)) {
1948 - $debug("Returning content from generic selector: " . $selector);
1949 - return $content;
1950 - }
1951 - }
1952 - }
1953 -
1954 - // Paragraph-based content detection - find regions with substantial text
1955 - $debug("Trying paragraph-based content detection");
1956 - $paragraphs = $xpath->query('//p[string-length(normalize-space()) > 50]');
1957 - if ($paragraphs && $paragraphs->length >= 3) {
1958 - $debug("Found " . $paragraphs->length . " substantial paragraphs");
1959 - // Collect all substantial paragraphs and their content
1960 - $paragraph_content = '';
1961 - foreach ($paragraphs as $p) {
1962 - $paragraph_content .= $dom->saveHTML($p) . "\n";
1963 - }
1964 - if (!empty($paragraph_content)) {
1965 - $debug("Returning paragraph-based content");
1966 - return $paragraph_content;
1967 - }
1968 - }
1969 -
1970 - // Improved body fallback - strip nav/header/footer elements first
1971 - $debug("Using improved body fallback");
1661 +
1662 + // Fallback: Return the body content if no specific selector matches
1663 + $debug("Using body fallback");
1972 1664 $body = $dom->getElementsByTagName('body');
1973 1665 if ($body->length > 0) {
1974 - // Clone the body to avoid modifying the original DOM
1975 - $body_clone = $body->item(0)->cloneNode(true);
1976 -
1977 - // Remove common non-content elements by tag name
1978 - $remove_tags = ['nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript'];
1979 - foreach ($remove_tags as $tag) {
1980 - $elements = $body_clone->getElementsByTagName($tag);
1981 - // Iterate backwards to safely remove elements
1982 - for ($i = $elements->length - 1; $i >= 0; $i--) {
1983 - $el = $elements->item($i);
1984 - if ($el && $el->parentNode) {
1985 - $el->parentNode->removeChild($el);
1986 - }
1987 - }
1988 - }
1989 -
1990 - // Remove elements with common non-content class names using XPath on the cloned body
1991 - $temp_dom = new DOMDocument();
1992 - @$temp_dom->appendChild($temp_dom->importNode($body_clone, true));
1993 - $temp_xpath = new DOMXPath($temp_dom);
1994 -
1995 - $remove_class_patterns = [
1996 - '//*[contains(@class, "nav")]',
1997 - '//*[contains(@class, "menu")]',
1998 - '//*[contains(@class, "sidebar")]',
1999 - '//*[contains(@class, "footer")]',
2000 - '//*[contains(@class, "header")]',
2001 - '//*[contains(@id, "nav")]',
2002 - '//*[contains(@id, "menu")]',
2003 - '//*[contains(@id, "sidebar")]',
2004 - '//*[contains(@id, "footer")]',
2005 - '//*[contains(@id, "header")]',
2006 - ];
2007 -
2008 - foreach ($remove_class_patterns as $pattern) {
2009 - $elements = $temp_xpath->query($pattern);
2010 - if ($elements) {
2011 - for ($i = $elements->length - 1; $i >= 0; $i--) {
2012 - $el = $elements->item($i);
2013 - if ($el && $el->parentNode) {
2014 - $el->parentNode->removeChild($el);
2015 - }
2016 - }
2017 - }
2018 - }
2019 -
2020 - $cleaned_content = $temp_dom->saveHTML();
2021 - if (!empty($cleaned_content)) {
2022 - $debug("Returning cleaned body content");
2023 - return $cleaned_content;
2024 - }
1666 + return $dom->saveHTML($body->item(0));
2025 1667 }
2026 -
1668 +
2027 1669 // Last resort: return the original HTML
2028 1670 $debug("Returning original HTML");
2029 1671 return $html;
2030 1672 } catch (Exception $e) {
@@ -2076,41 +1718,47 @@
2076 1718 try {
2077 1719 // Verify the request
2078 1720 check_ajax_referer('mxchat_status_nonce', 'nonce');
2079 1721
2080 - // Get active queue IDs
2081 - $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
2082 - $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
1722 + // Get the status just like in your admin page
1723 + $pdf_url = get_transient('mxchat_last_pdf_url');
1724 + $sitemap_url = get_transient('mxchat_last_sitemap_url');
2083 1725
2084 - $sitemap_status = false;
2085 - $pdf_status = false;
1726 + $pdf_status = $pdf_url ? $this->mxchat_get_pdf_processing_status($pdf_url) : false;
1727 + $sitemap_status = $sitemap_url ? $this->mxchat_get_sitemap_processing_status($sitemap_url) : false;
2086 1728
2087 - // Get sitemap queue status
2088 - if ($sitemap_queue_id) {
2089 - $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
1729 + // Add the PDF URL to the status object
1730 + if ($pdf_status && $pdf_url) {
1731 + $pdf_status['pdf_url'] = $pdf_url;
2090 1732 }
2091 1733
2092 - // Get PDF queue status
2093 - if ($pdf_queue_id) {
2094 - $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
2095 - }
1734 + // Set the current PDF URL for the manual batch processing button
1735 + $current_pdf_url = $pdf_url;
2096 1736
1737 + // Check for true processing status, not just presence of status
2097 1738 $is_active_processing =
2098 - ($sitemap_status && $sitemap_status['status'] === 'processing') ||
2099 - ($pdf_status && $pdf_status['status'] === 'processing');
1739 + ($sitemap_status && isset($sitemap_status['status']) && $sitemap_status['status'] === 'processing') ||
1740 + ($pdf_status && isset($pdf_status['status']) && $pdf_status['status'] === 'processing');
2100 1741
1742 + // Get single URL status, but only if no processing is active
1743 + $single_url_status = !$is_active_processing ? $this->mxchat_get_single_url_status() : false;
1744 +
1745 + // REMOVED: Auto-clearing of completed status - now only done via dismiss button
1746 +
2101 1747 // Return JSON response with the status data
2102 1748 wp_send_json(array(
2103 1749 'pdf_status' => $pdf_status,
2104 1750 'sitemap_status' => $sitemap_status,
1751 + 'single_url_status' => $single_url_status,
2105 1752 'is_processing' => $is_active_processing,
2106 - 'sitemap_queue_id' => $sitemap_queue_id,
2107 - 'pdf_queue_id' => $pdf_queue_id
1753 + 'current_pdf_url' => $current_pdf_url
2108 1754 ));
2109 1755
2110 1756 } catch (Exception $e) {
1757 + // Log the error
2111 1758 //error_log('MxChat Status Update Error: ' . $e->getMessage());
2112 1759
1760 + // Return a friendly error response
2113 1761 wp_send_json_error(array(
2114 1762 'message' => 'Error getting status updates: ' . $e->getMessage(),
2115 1763 'status' => 'error'
2116 1764 ));
@@ -2115,1371 +1763,8 @@
2115 1763 'status' => 'error'
2116 1764 ));
2117 1765 }
2118 1766 }
2119 -
2120 -/**
2121 - * Helper function to get queue status data
2122 - */
2123 -private function mxchat_get_queue_status_data($queue_id, $type = 'sitemap') {
2124 - global $wpdb;
2125 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
2126 -
2127 - // Get counts by status
2128 - $counts = $wpdb->get_results($wpdb->prepare(
2129 - "SELECT status, COUNT(*) as count
2130 - FROM $table_name
2131 - WHERE queue_id = %s
2132 - GROUP BY status",
2133 - $queue_id
2134 - ), OBJECT_K);
2135 -
2136 - $total = 0;
2137 - $completed = 0;
2138 - $failed = 0;
2139 - $processing = 0;
2140 - $pending = 0;
2141 -
2142 - foreach ($counts as $status => $data) {
2143 - $count = absint($data->count);
2144 - $total += $count;
2145 -
2146 - switch ($status) {
2147 - case 'completed':
2148 - $completed = $count;
2149 - break;
2150 - case 'failed':
2151 - $failed = $count;
2152 - break;
2153 - case 'processing':
2154 - $processing = $count;
2155 - break;
2156 - case 'pending':
2157 - $pending = $count;
2158 - break;
2159 - }
2160 - }
2161 -
2162 - if ($total === 0) {
2163 - return false;
2164 - }
2165 -
2166 - // Calculate percentage
2167 - $percentage = round((($completed + $failed) / $total) * 100);
2168 -
2169 - // Get failed items details (limit to 50)
2170 - $failed_items = array();
2171 - if ($failed > 0) {
2172 - $failed_results = $wpdb->get_results($wpdb->prepare(
2173 - "SELECT item_type, item_data, error_message, attempts, completed_at
2174 - FROM $table_name
2175 - WHERE queue_id = %s
2176 - AND status = 'failed'
2177 - AND attempts >= max_attempts
2178 - ORDER BY id DESC
2179 - LIMIT 50",
2180 - $queue_id
2181 - ));
2182 -
2183 - foreach ($failed_results as $item) {
2184 - $data = json_decode($item->item_data, true);
2185 - $url = $type === 'pdf' ? ($data['pdf_url'] ?? '') : ($data['url'] ?? '');
2186 -
2187 - $failed_items[] = array(
2188 - 'url' => $url,
2189 - 'page' => $type === 'pdf' ? ($data['page_number'] ?? 0) : 0,
2190 - 'error' => $item->error_message,
2191 - 'retries' => $item->attempts,
2192 - 'time' => strtotime($item->completed_at)
2193 - );
2194 - }
2195 - }
2196 -
2197 - // Get queue metadata
2198 - $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
2199 -
2200 - // Determine if queue is complete
2201 - $is_complete = ($pending === 0 && $processing === 0);
2202 -
2203 - // Get last update time
2204 - $last_update = $wpdb->get_var($wpdb->prepare(
2205 - "SELECT MAX(UNIX_TIMESTAMP(COALESCE(completed_at, started_at, created_at)))
2206 - FROM $table_name
2207 - WHERE queue_id = %s",
2208 - $queue_id
2209 - ));
2210 -
2211 - $last_update_text = $last_update ? human_time_diff($last_update, time()) . ' ' . __('ago', 'mxchat') : __('Just now', 'mxchat');
2212 -
2213 - // Format based on type
2214 - if ($type === 'pdf') {
2215 - return array(
2216 - 'total_pages' => $total,
2217 - 'processed_pages' => $completed + $failed,
2218 - 'failed_pages' => $failed,
2219 - 'percentage' => $percentage,
2220 - 'status' => $is_complete ? 'complete' : 'processing',
2221 - 'last_update' => $last_update_text,
2222 - 'failed_pages_list' => $failed_items,
2223 - 'pdf_url' => $source_url,
2224 - 'queue_id' => $queue_id
2225 - );
2226 - } else {
2227 - return array(
2228 - 'total_urls' => $total,
2229 - 'processed_urls' => $completed + $failed,
2230 - 'failed_urls' => $failed,
2231 - 'percentage' => $percentage,
2232 - 'status' => $is_complete ? 'complete' : 'processing',
2233 - 'last_update' => $last_update_text,
2234 - 'failed_urls_list' => $failed_items,
2235 - 'sitemap_url' => $source_url,
2236 - 'queue_id' => $queue_id
2237 - );
2238 - }
2239 -}
2240 -
2241 -/**
2242 - * Public method to get processing status for both sitemap and PDF queues
2243 - * Used by admin pages to display processing status
2244 - *
2245 - * @return array Array with 'sitemap_status', 'pdf_status', and 'is_processing' keys
2246 - */
2247 -public function mxchat_get_processing_statuses() {
2248 - $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
2249 - $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
2250 -
2251 - $sitemap_status = false;
2252 - $pdf_status = false;
2253 -
2254 - if ($sitemap_queue_id) {
2255 - $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
2256 - }
2257 -
2258 - if ($pdf_queue_id) {
2259 - $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
2260 - }
2261 -
2262 - $is_processing =
2263 - ($sitemap_status && $sitemap_status['status'] === 'processing') ||
2264 - ($pdf_status && $pdf_status['status'] === 'processing');
2265 -
2266 - return array(
2267 - 'sitemap_status' => $sitemap_status,
2268 - 'pdf_status' => $pdf_status,
2269 - 'is_processing' => $is_processing
2270 - );
2271 -}
2272 -
2273 -/**
2274 - * AJAX handler to get recent knowledge entries for real-time table updates
2275 - * UPDATED: Now supports both WordPress DB and Pinecone data sources
2276 - */
2277 -public function ajax_mxchat_get_recent_entries() {
2278 - check_ajax_referer('mxchat_entries_nonce', 'nonce');
2279 -
2280 - if (!current_user_can('manage_options')) {
2281 - wp_send_json_error(array('message' => 'Unauthorized'));
2282 - return;
2283 - }
2284 -
2285 - global $wpdb;
2286 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2287 -
2288 - // Get parameters
2289 - $last_id = isset($_POST['last_id']) ? absint($_POST['last_id']) : 0;
2290 - $limit = isset($_POST['limit']) ? min(absint($_POST['limit']), 50) : 10;
2291 - $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2292 -
2293 - // Check if Pinecone is enabled for this bot
2294 - $pinecone_manager = $this->mxchat_get_pinecone_manager();
2295 - $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
2296 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2297 - $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
2298 -
2299 - if ($use_pinecone && $has_pinecone_api) {
2300 - // PINECONE DATA SOURCE - Get grouped entry count (not raw vector count)
2301 - // Use mxchat_fetch_pinecone_records which returns total_unique_entries
2302 - $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, '', 1, 10, $bot_id, '');
2303 - $total_count = $records['total'] ?? 0;
2304 -
2305 - // For Pinecone, we don't return individual entries during polling
2306 - // (entries are already displayed on page load via mxchat_fetch_pinecone_records)
2307 - // We just return the updated count
2308 - wp_send_json_success(array(
2309 - 'entries' => array(),
2310 - 'total_count' => absint($total_count),
2311 - 'max_id' => $last_id,
2312 - 'data_source' => 'pinecone'
2313 - ));
2314 - return;
2315 - }
2316 -
2317 - // WORDPRESS DB DATA SOURCE
2318 - // Build query to get entries newer than last_id
2319 - $where_clauses = array('1=1');
2320 - $where_values = array();
2321 -
2322 - if ($last_id > 0) {
2323 - $where_clauses[] = 'id > %d';
2324 - $where_values[] = $last_id;
2325 - }
2326 -
2327 - // Note: WordPress DB table doesn't have bot_id column
2328 - // Multi-bot filtering is handled via Pinecone namespaces
2329 -
2330 - $where_sql = implode(' AND ', $where_clauses);
2331 -
2332 - // Get recent entries
2333 - $query = "SELECT id, article_content, source_url, timestamp
2334 - FROM $table_name
2335 - WHERE $where_sql
2336 - ORDER BY id DESC
2337 - LIMIT %d";
2338 -
2339 - $where_values[] = $limit;
2340 -
2341 - $entries = $wpdb->get_results($wpdb->prepare($query, $where_values));
2342 -
2343 - // Get total count of GROUPED entries (by source_url) - matches pagination display
2344 - // Count unique source_urls (excluding mxchat:// internal refs) + count of ungrouped rows
2345 - $total_count = $wpdb->get_var(
2346 - "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2347 - (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
2348 - );
2349 -
2350 - // Format entries for response
2351 - $formatted_entries = array();
2352 - $preview_length = 150;
2353 - foreach ($entries as $entry) {
2354 - // Parse chunk metadata using the proper chunker method (same as initial page load)
2355 - if (class_exists('MxChat_Chunker')) {
2356 - $chunk_meta = MxChat_Chunker::parse_stored_chunk($entry->article_content);
2357 - $display_content = $chunk_meta['text'];
2358 - $chunk_metadata = $chunk_meta['metadata'];
2359 - } else {
2360 - $display_content = $entry->article_content;
2361 - $chunk_metadata = array();
2362 - }
2363 -
2364 - $content_preview = mb_strlen($display_content) > $preview_length
2365 - ? mb_substr($display_content, 0, $preview_length) . '...'
2366 - : $display_content;
2367 -
2368 - $formatted_entries[] = array(
2369 - 'id' => $entry->id,
2370 - 'preview' => esc_html($content_preview),
2371 - 'full_content' => wp_kses_post(wpautop($display_content)),
2372 - 'content_length' => mb_strlen($display_content),
2373 - 'preview_length' => $preview_length,
2374 - 'source_url' => $entry->source_url,
2375 - 'has_link' => !empty($entry->source_url) && strpos($entry->source_url, 'mxchat://') !== 0,
2376 - 'chunk_metadata' => $chunk_metadata,
2377 - 'bot_id' => $entry->bot_id ?? 'default',
2378 - 'edit_nonce' => wp_create_nonce('mxchat_edit_entry_nonce'),
2379 - 'delete_nonce' => wp_create_nonce('mxchat_delete_prompt_nonce')
2380 - );
2381 - }
2382 -
2383 - wp_send_json_success(array(
2384 - 'entries' => $formatted_entries,
2385 - 'total_count' => absint($total_count),
2386 - 'max_id' => !empty($entries) ? $entries[0]->id : $last_id,
2387 - 'data_source' => 'wordpress'
2388 - ));
2389 -}
2390 -
2391 -/**
2392 - * Get Pinecone total count from stats API
2393 - * Helper function for ajax_mxchat_get_recent_entries
2394 - */
2395 -private function mxchat_get_pinecone_count_from_stats($pinecone_options) {
2396 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2397 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
2398 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
2399 -
2400 - if (empty($api_key) || empty($host)) {
2401 - return 0;
2402 - }
2403 -
2404 - try {
2405 - $stats_url = "https://{$host}/describe_index_stats";
2406 -
2407 - $response = wp_remote_post($stats_url, array(
2408 - 'headers' => array(
2409 - 'Api-Key' => $api_key,
2410 - 'Content-Type' => 'application/json'
2411 - ),
2412 - 'body' => '{}',
2413 - 'timeout' => 10
2414 - ));
2415 -
2416 - if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2417 - $body = wp_remote_retrieve_body($response);
2418 - $stats_data = json_decode($body, true);
2419 -
2420 - // If namespace is specified, get count from that specific namespace
2421 - if (!empty($namespace) && isset($stats_data['namespaces'][$namespace]['vectorCount'])) {
2422 - return intval($stats_data['namespaces'][$namespace]['vectorCount']);
2423 - }
2424 -
2425 - // If no namespace specified or namespace not found in response, use total
2426 - return intval($stats_data['totalVectorCount'] ?? 0);
2427 - }
2428 -
2429 - return 0;
2430 -
2431 - } catch (Exception $e) {
2432 - return 0;
2433 - }
2434 -}
2435 -
2436 -/**
2437 - * AJAX handler to refresh Pinecone entries table via AJAX
2438 - * Returns the table HTML for updating the UI without a full page reload
2439 - */
2440 -public function ajax_mxchat_refresh_pinecone_entries() {
2441 - check_ajax_referer('mxchat_entries_nonce', 'nonce');
2442 -
2443 - if (!current_user_can('manage_options')) {
2444 - wp_send_json_error(array('message' => 'Unauthorized'));
2445 - return;
2446 - }
2447 -
2448 - $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2449 - $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2450 - $per_page = 25;
2451 - $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2452 - $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2453 -
2454 - // Get Pinecone manager and options
2455 - $pinecone_manager = $this->mxchat_get_pinecone_manager();
2456 - if (!$pinecone_manager) {
2457 - wp_send_json_error(array('message' => 'Pinecone manager not available'));
2458 - return;
2459 - }
2460 -
2461 - $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
2462 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2463 - $pinecone_api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2464 -
2465 - if (!$use_pinecone || empty($pinecone_api_key)) {
2466 - wp_send_json_error(array('message' => 'Pinecone not configured'));
2467 - return;
2468 - }
2469 -
2470 - // Fetch records from Pinecone
2471 - $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, $search_query, $page, $per_page, $bot_id, $content_type_filter);
2472 - $prompts = $records['data'] ?? array();
2473 - $total_records = $records['total'] ?? 0;
2474 -
2475 - // Preprocess Pinecone records — set chunk_metadata and display_content
2476 - // (matches admin-knowledge-page.php preprocessing)
2477 - foreach ($prompts as $prompt) {
2478 - if (isset($prompt->chunk_index) && $prompt->chunk_index !== null) {
2479 - $prompt->chunk_metadata = array(
2480 - 'chunk_index' => intval($prompt->chunk_index),
2481 - 'total_chunks' => isset($prompt->total_chunks) ? intval($prompt->total_chunks) : null,
2482 - 'is_chunked' => isset($prompt->is_chunked) ? (bool) $prompt->is_chunked : true,
2483 - 'source_url' => $prompt->source_url ?? ''
2484 - );
2485 - $prompt->display_content = $prompt->article_content;
2486 - } else {
2487 - $prompt->chunk_metadata = array();
2488 - $prompt->display_content = $prompt->article_content ?? '';
2489 - }
2490 - }
2491 -
2492 - // Group prompts by source_url
2493 - $grouped_prompts = array();
2494 - foreach ($prompts as $prompt) {
2495 - $source_url = '';
2496 - if (!empty($prompt->chunk_metadata['source_url'])) {
2497 - $source_url = $prompt->chunk_metadata['source_url'];
2498 - } elseif (!empty($prompt->source_url)) {
2499 - $source_url = $prompt->source_url;
2500 - }
2501 -
2502 - if (!empty($source_url)) {
2503 - if (!isset($grouped_prompts[$source_url])) {
2504 - $grouped_prompts[$source_url] = array();
2505 - }
2506 - $grouped_prompts[$source_url][] = $prompt;
2507 - } else {
2508 - $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2509 - }
2510 - }
2511 -
2512 - // Sort each group by chunk_index
2513 - foreach ($grouped_prompts as $source_url => &$group) {
2514 - usort($group, function($a, $b) {
2515 - $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2516 - $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2517 - return $index_a - $index_b;
2518 - });
2519 - }
2520 - unset($group);
2521 -
2522 - // Build HTML for the table rows - must match admin-knowledge-page.php structure exactly
2523 - ob_start();
2524 - $display_index = 0;
2525 - $current_page = $page;
2526 - $data_source = 'pinecone';
2527 - $current_bot_id = $bot_id;
2528 - $preview_length = 150;
2529 -
2530 - if (empty($grouped_prompts)) {
2531 - echo '<tr><td colspan="4" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2532 - esc_html_e('No knowledge entries found in Pinecone.', 'mxchat');
2533 - echo '</td></tr>';
2534 - } else {
2535 - foreach ($grouped_prompts as $source_url => $group) {
2536 - $chunk_count = count($group);
2537 - $first_prompt = $group[0];
2538 - $display_index++;
2539 -
2540 - if ($chunk_count > 1) {
2541 - // Multiple chunks - show grouped row with expand button
2542 - $group_id = 'group-' . md5($source_url);
2543 - ?>
2544 - <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2545 - class="mxchat-chunk-group-header"
2546 - data-source="<?php echo esc_attr($data_source); ?>"
2547 - data-group-id="<?php echo esc_attr($group_id); ?>"
2548 - style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2549 - <td style="padding: 12px 16px; text-align: center;">
2550 - <input type="checkbox"
2551 - class="mxchat-entry-checkbox"
2552 - data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2553 - data-source="<?php echo esc_attr($data_source); ?>"
2554 - data-source-url="<?php echo esc_attr($source_url); ?>"
2555 - data-is-group="true"
2556 - data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2557 - </td>
2558 - <td style="padding: 12px 16px; font-size: 13px;">
2559 - <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2560 - </td>
2561 - <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2562 - <div class="mxchat-chunk-group-info">
2563 - <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2564 - <span class="dashicons dashicons-arrow-right-alt2"></span>
2565 - </button>
2566 - <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2567 - <span class="mxchat-chunk-preview">
2568 - <?php
2569 - $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : (isset($first_prompt->article_content) ? $first_prompt->article_content : '');
2570 - $content_preview = mb_substr($parent_content, 0, 100);
2571 - echo esc_html($content_preview . '...');
2572 - ?>
2573 - </span>
2574 - </div>
2575 - </td>
2576 - <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2577 - <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2578 - <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2579 - <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2580 - <?php esc_html_e('View Source', 'mxchat'); ?>
2581 - </a>
2582 - <?php else : ?>
2583 - <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2584 - <?php endif; ?>
2585 - </td>
2586 - <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2587 - <?php if ($data_source !== 'pinecone') : ?>
2588 - <button type="button"
2589 - class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2590 - data-source-url="<?php echo esc_attr($source_url); ?>"
2591 - data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2592 - data-data-source="<?php echo esc_attr($data_source); ?>"
2593 - data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2594 - data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2595 - title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2596 - <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2597 - </button>
2598 - <?php endif; ?>
2599 - <button type="button"
2600 - class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2601 - data-source-url="<?php echo esc_attr($source_url); ?>"
2602 - data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2603 - data-data-source="<?php echo esc_attr($data_source); ?>"
2604 - data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2605 - data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2606 - style="color: var(--mxch-error);"
2607 - title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2608 - <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2609 - </button>
2610 - </td>
2611 - </tr>
2612 - <?php
2613 - // Render hidden chunk rows
2614 - foreach ($group as $chunk_index => $chunk) {
2615 - $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
2616 - $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
2617 - $content = isset($chunk->display_content) ? $chunk->display_content : (isset($chunk->article_content) ? $chunk->article_content : '');
2618 - $content_preview = mb_strlen($content) > $preview_length
2619 - ? mb_substr($content, 0, $preview_length) . '...'
2620 - : $content;
2621 - ?>
2622 - <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
2623 - class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
2624 - data-source="<?php echo esc_attr($data_source); ?>"
2625 - style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
2626 - <td style="padding: 12px 16px; text-align: center;">
2627 - <!-- Checkbox column placeholder for chunks (managed by group) -->
2628 - </td>
2629 - <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
2630 - <!-- Hidden ID column for chunks -->
2631 - </td>
2632 - <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2633 - <div class="mxchat-accordion-wrapper">
2634 - <div class="mxchat-content-preview">
2635 - <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
2636 - <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
2637 - </span>
2638 - <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2639 - <?php if (mb_strlen($content) > $preview_length) : ?>
2640 - <button class="mxchat-expand-toggle" type="button">
2641 - <span class="dashicons dashicons-arrow-down-alt2"></span>
2642 - </button>
2643 - <?php endif; ?>
2644 - </div>
2645 - <div class="mxchat-content-full" style="display: none;">
2646 - <div class="content-view">
2647 - <?php
2648 - if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2649 - echo '<div dir="rtl" lang="he" class="rtl-content">';
2650 - echo wp_kses_post(wpautop($content));
2651 - echo '</div>';
2652 - } else {
2653 - echo wp_kses_post(wpautop($content));
2654 - }
2655 - ?>
2656 - </div>
2657 - </div>
2658 - </div>
2659 - </td>
2660 - <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2661 - <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
2662 - </td>
2663 - <td class="mxchat-actions-cell" style="padding: 12px 16px;">
2664 - <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
2665 - </td>
2666 - </tr>
2667 - <?php
2668 - }
2669 - } else {
2670 - // Single entry - display normally with accordion
2671 - $prompt = $first_prompt;
2672 - $content = isset($prompt->display_content) ? $prompt->display_content : (isset($prompt->article_content) ? $prompt->article_content : '');
2673 - $content_preview = mb_strlen($content) > $preview_length
2674 - ? mb_substr($content, 0, $preview_length) . '...'
2675 - : $content;
2676 - ?>
2677 - <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
2678 - data-source="<?php echo esc_attr($data_source); ?>"
2679 - style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2680 - <td style="padding: 12px 16px; text-align: center;">
2681 - <input type="checkbox"
2682 - class="mxchat-entry-checkbox"
2683 - data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2684 - data-source="<?php echo esc_attr($data_source); ?>"
2685 - data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
2686 - data-is-group="false"
2687 - data-chunk-count="1">
2688 - </td>
2689 - <td style="padding: 12px 16px; font-size: 13px;">
2690 - <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2691 - </td>
2692 - <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2693 - <div class="mxchat-accordion-wrapper">
2694 - <div class="mxchat-content-preview">
2695 - <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2696 - <?php if (mb_strlen($content) > $preview_length) : ?>
2697 - <button class="mxchat-expand-toggle" type="button">
2698 - <span class="dashicons dashicons-arrow-down-alt2"></span>
2699 - </button>
2700 - <?php endif; ?>
2701 - </div>
2702 - <div class="mxchat-content-full" style="display: none;">
2703 - <div class="content-view">
2704 - <?php
2705 - if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2706 - echo '<div dir="rtl" lang="he" class="rtl-content">';
2707 - echo wp_kses_post(wpautop($content));
2708 - echo '</div>';
2709 - } else {
2710 - echo wp_kses_post(wpautop($content));
2711 - }
2712 - ?>
2713 - </div>
2714 - </div>
2715 - </div>
2716 - </td>
2717 - <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2718 - <?php
2719 - $actual_source = $source_url;
2720 - if (strpos($source_url, '_ungrouped_') === 0) {
2721 - $actual_source = $prompt->source_url ?? '';
2722 - }
2723 - if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
2724 - <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2725 - <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2726 - <?php esc_html_e('View', 'mxchat'); ?>
2727 - </a>
2728 - <?php else : ?>
2729 - <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
2730 - <?php endif; ?>
2731 - </td>
2732 - <td style="padding: 12px 16px;">
2733 - <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-ajax" data-vector-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_pinecone_prompt_nonce'); ?>" style="color: var(--mxch-error);">
2734 - <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2735 - </button>
2736 - </td>
2737 - </tr>
2738 - <?php
2739 - }
2740 - }
2741 - }
2742 - $html = ob_get_clean();
2743 -
2744 - // Generate pagination HTML for Pinecone
2745 - $total_pages = ceil($total_records / $per_page);
2746 - $pagination_html = '';
2747 - if ($total_pages > 1) {
2748 - $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
2749 -
2750 - // Previous button
2751 - if ($page > 1) {
2752 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
2753 - }
2754 -
2755 - // Page numbers
2756 - $start_page = max(1, $page - 2);
2757 - $end_page = min($total_pages, $page + 2);
2758 -
2759 - if ($start_page > 1) {
2760 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
2761 - if ($start_page > 2) {
2762 - $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2763 - }
2764 - }
2765 -
2766 - for ($i = $start_page; $i <= $end_page; $i++) {
2767 - if ($i == $page) {
2768 - $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
2769 - } else {
2770 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
2771 - }
2772 - }
2773 -
2774 - if ($end_page < $total_pages) {
2775 - if ($end_page < $total_pages - 1) {
2776 - $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2777 - }
2778 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
2779 - }
2780 -
2781 - // Next button
2782 - if ($page < $total_pages) {
2783 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
2784 - }
2785 -
2786 - $pagination_html .= '</div>';
2787 - }
2788 -
2789 - wp_send_json_success(array(
2790 - 'html' => $html,
2791 - 'pagination_html' => $pagination_html,
2792 - 'total_count' => $total_records,
2793 - 'total_pages' => $total_pages,
2794 - 'page' => $page,
2795 - 'per_page' => $per_page,
2796 - 'data_source' => 'pinecone'
2797 - ));
2798 -}
2799 -
2800 -/**
2801 - * AJAX handler for pagination - handles both WordPress DB and Pinecone data sources
2802 - * Returns paginated entries without requiring a full page reload
2803 - */
2804 -public function ajax_mxchat_paginate_entries() {
2805 - check_ajax_referer('mxchat_entries_nonce', 'nonce');
2806 -
2807 - if (!current_user_can('manage_options')) {
2808 - wp_send_json_error(array('message' => 'Unauthorized'));
2809 - return;
2810 - }
2811 -
2812 - $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2813 - $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2814 - $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2815 - $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2816 - $per_page = 25;
2817 -
2818 - // Check if Pinecone is enabled for this bot
2819 - $pinecone_manager = $this->mxchat_get_pinecone_manager();
2820 - $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
2821 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2822 - $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
2823 -
2824 - if ($use_pinecone && $has_pinecone_api) {
2825 - // Delegate to Pinecone pagination handler (pass search params)
2826 - $_POST['page'] = $page;
2827 - $_POST['search'] = $search_query;
2828 - $_POST['content_type'] = $content_type_filter;
2829 - $this->ajax_mxchat_refresh_pinecone_entries();
2830 - return;
2831 - }
2832 -
2833 - // WordPress DB pagination - MUST match initial page load logic exactly
2834 - global $wpdb;
2835 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2836 - $offset = ($page - 1) * $per_page;
2837 -
2838 - // Build WHERE clause for search and content type filtering
2839 - $where_clauses = array();
2840 - $where_values = array();
2841 -
2842 - if ($search_query) {
2843 - $where_clauses[] = "article_content LIKE %s";
2844 - $where_values[] = '%' . $wpdb->esc_like($search_query) . '%';
2845 - }
2846 -
2847 - if ($content_type_filter) {
2848 - switch ($content_type_filter) {
2849 - case 'manual':
2850 - $where_clauses[] = "(source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')";
2851 - break;
2852 - case 'pdf':
2853 - $where_clauses[] = "source_url LIKE '%.pdf'";
2854 - break;
2855 - case 'url':
2856 - $where_clauses[] = "source_url != '' AND source_url IS NOT NULL AND source_url NOT LIKE 'mxchat://%' AND source_url NOT LIKE '%.pdf'";
2857 - break;
2858 - }
2859 - }
2860 -
2861 - $where_sql = !empty($where_clauses) ? 'WHERE ' . implode(' AND ', $where_clauses) : '';
2862 -
2863 - // Count grouped entries with filters applied
2864 - if (!empty($where_values)) {
2865 - $count_args = array_merge($where_values, $where_values);
2866 - $count_query = $wpdb->prepare(
2867 - "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2868 - (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))",
2869 - ...$count_args
2870 - );
2871 - $total_records = $wpdb->get_var($count_query);
2872 - } else if (!empty($where_sql)) {
2873 - // Content type filter only (no search), no prepared values needed
2874 - $total_records = $wpdb->get_var(
2875 - "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2876 - (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))"
2877 - );
2878 - } else {
2879 - // No filters
2880 - $total_records = $wpdb->get_var(
2881 - "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2882 - (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
2883 - );
2884 - }
2885 - $total_pages = ceil($total_records / $per_page);
2886 -
2887 - // Step 1: Get unique source_urls for this page (ordered by latest timestamp) with filters
2888 - if (!empty($where_values)) {
2889 - $query_args = array_merge($where_values, array($per_page, $offset));
2890 - $urls_query = $wpdb->prepare(
2891 - "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2892 - {$where_sql}
2893 - GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2894 - ...$query_args
2895 - );
2896 - } else if (!empty($where_sql)) {
2897 - $urls_query = $wpdb->prepare(
2898 - "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2899 - {$where_sql}
2900 - GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2901 - $per_page, $offset
2902 - );
2903 - } else {
2904 - $urls_query = $wpdb->prepare(
2905 - "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2906 - GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2907 - $per_page, $offset
2908 - );
2909 - }
2910 - $page_urls = $wpdb->get_results($urls_query);
2911 -
2912 - // Step 2: Build list of source_urls to fetch
2913 - $url_list = array();
2914 - $url_order_map = array();
2915 - $order_index = 0;
2916 - foreach ($page_urls as $url_row) {
2917 - $url = $url_row->source_url;
2918 - $url_list[] = $url;
2919 - $url_order_map[$url] = $order_index++;
2920 - }
2921 -
2922 - // Step 3: Fetch all rows for these source_urls (with search filter if applicable)
2923 - $prompts = array();
2924 - if (!empty($url_list)) {
2925 - $placeholders = implode(',', array_fill(0, count($url_list), '%s'));
2926 - if ($search_query) {
2927 - // Include search filter in the final fetch
2928 - $prompts_query = $wpdb->prepare(
2929 - "SELECT id, article_content, source_url, timestamp, role_restriction
2930 - FROM {$table_name}
2931 - WHERE source_url IN ($placeholders) AND article_content LIKE %s
2932 - ORDER BY timestamp DESC",
2933 - ...array_merge($url_list, ['%' . $wpdb->esc_like($search_query) . '%'])
2934 - );
2935 - } else {
2936 - $prompts_query = $wpdb->prepare(
2937 - "SELECT id, article_content, source_url, timestamp, role_restriction
2938 - FROM {$table_name}
2939 - WHERE source_url IN ($placeholders)
2940 - ORDER BY timestamp DESC",
2941 - $url_list
2942 - );
2943 - }
2944 - $prompts = $wpdb->get_results($prompts_query);
2945 - }
2946 -
2947 - // Group prompts by source_url for chunk display
2948 - $grouped_prompts = array();
2949 - foreach ($prompts as $prompt) {
2950 - $source_url = $prompt->source_url ?? '';
2951 -
2952 - // Parse chunk metadata using the proper chunker method (same as initial page load)
2953 - if (class_exists('MxChat_Chunker')) {
2954 - $chunk_meta = MxChat_Chunker::parse_stored_chunk($prompt->article_content);
2955 - $prompt->chunk_metadata = $chunk_meta['metadata'];
2956 - $prompt->display_content = $chunk_meta['text'];
2957 - } else {
2958 - $prompt->chunk_metadata = array();
2959 - $prompt->display_content = $prompt->article_content;
2960 - }
2961 -
2962 - if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0) {
2963 - if (!isset($grouped_prompts[$source_url])) {
2964 - $grouped_prompts[$source_url] = array();
2965 - }
2966 - $grouped_prompts[$source_url][] = $prompt;
2967 - } else {
2968 - // Ungrouped entries
2969 - $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2970 - }
2971 - }
2972 -
2973 - // Sort groups by the original URL order (newest first)
2974 - uksort($grouped_prompts, function($a, $b) use ($url_order_map) {
2975 - $order_a = isset($url_order_map[$a]) ? $url_order_map[$a] : PHP_INT_MAX;
2976 - $order_b = isset($url_order_map[$b]) ? $url_order_map[$b] : PHP_INT_MAX;
2977 - return $order_a - $order_b;
2978 - });
2979 -
2980 - // Sort each group internally by chunk_index
2981 - foreach ($grouped_prompts as $source_url => &$group) {
2982 - usort($group, function($a, $b) {
2983 - $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2984 - $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2985 - return $index_a - $index_b;
2986 - });
2987 - }
2988 - unset($group);
2989 -
2990 - // Build HTML for the table rows
2991 - ob_start();
2992 - $display_index = 0;
2993 - $current_page = $page;
2994 - $data_source = 'wordpress';
2995 - $current_bot_id = $bot_id;
2996 - $preview_length = 150;
2997 -
2998 - if (empty($grouped_prompts)) {
2999 - echo '<tr><td colspan="5" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
3000 - esc_html_e('No knowledge entries found. Use the Import Options to add content.', 'mxchat');
3001 - echo '</td></tr>';
3002 - } else {
3003 - foreach ($grouped_prompts as $source_url => $group) {
3004 - $chunk_count = count($group);
3005 - $first_prompt = $group[0];
3006 - $display_index++;
3007 -
3008 - if ($chunk_count > 1) {
3009 - // Multiple chunks - show grouped row with expand button
3010 - $group_id = 'group-' . md5($source_url);
3011 - ?>
3012 - <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
3013 - class="mxchat-chunk-group-header"
3014 - data-source="<?php echo esc_attr($data_source); ?>"
3015 - data-group-id="<?php echo esc_attr($group_id); ?>"
3016 - style="border-bottom: 1px solid var(--mxch-card-border);">
3017 - <td style="padding: 12px 16px; text-align: center;">
3018 - <input type="checkbox"
3019 - class="mxchat-entry-checkbox"
3020 - data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
3021 - data-source="<?php echo esc_attr($data_source); ?>"
3022 - data-source-url="<?php echo esc_attr($source_url); ?>"
3023 - data-is-group="true"
3024 - data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
3025 - </td>
3026 - <td style="padding: 12px 16px; font-size: 13px;">
3027 - <?php echo esc_html($first_prompt->id); ?>
3028 - </td>
3029 - <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3030 - <div class="mxchat-chunk-group-info">
3031 - <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
3032 - <span class="dashicons dashicons-arrow-right-alt2"></span>
3033 - </button>
3034 - <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
3035 - <span class="mxchat-chunk-preview">
3036 - <?php
3037 - $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : $first_prompt->article_content;
3038 - $content_preview = mb_substr($parent_content, 0, 100);
3039 - echo esc_html($content_preview . '...');
3040 - ?>
3041 - </span>
3042 - </div>
3043 - </td>
3044 - <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3045 - <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
3046 - <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
3047 - <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
3048 - <?php esc_html_e('View Source', 'mxchat'); ?>
3049 - </a>
3050 - <?php else : ?>
3051 - <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
3052 - <?php endif; ?>
3053 - </td>
3054 - <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
3055 - <?php if ($data_source !== 'pinecone') : ?>
3056 - <button type="button"
3057 - class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
3058 - data-source-url="<?php echo esc_attr($source_url); ?>"
3059 - data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
3060 - data-data-source="<?php echo esc_attr($data_source); ?>"
3061 - data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3062 - data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
3063 - title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
3064 - <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
3065 - </button>
3066 - <?php endif; ?>
3067 - <button type="button"
3068 - class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
3069 - data-source-url="<?php echo esc_attr($source_url); ?>"
3070 - data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
3071 - data-data-source="<?php echo esc_attr($data_source); ?>"
3072 - data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3073 - data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
3074 - style="color: var(--mxch-error);"
3075 - title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
3076 - <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
3077 - </button>
3078 - </td>
3079 - </tr>
3080 - <?php
3081 - // Render hidden chunk rows
3082 - foreach ($group as $chunk_index => $chunk) {
3083 - $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
3084 - $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
3085 - $content = isset($chunk->display_content) ? $chunk->display_content : $chunk->article_content;
3086 - $content_preview = mb_strlen($content) > $preview_length
3087 - ? mb_substr($content, 0, $preview_length) . '...'
3088 - : $content;
3089 - ?>
3090 - <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
3091 - class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
3092 - data-source="<?php echo esc_attr($data_source); ?>"
3093 - style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
3094 - <td style="padding: 12px 16px; text-align: center;">
3095 - <!-- Checkbox column placeholder for chunks (managed by group) -->
3096 - </td>
3097 - <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
3098 - <!-- Hidden ID column for chunks -->
3099 - </td>
3100 - <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3101 - <div class="mxchat-accordion-wrapper">
3102 - <div class="mxchat-content-preview">
3103 - <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
3104 - <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
3105 - </span>
3106 - <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
3107 - <?php if (mb_strlen($content) > $preview_length) : ?>
3108 - <button class="mxchat-expand-toggle" type="button">
3109 - <span class="dashicons dashicons-arrow-down-alt2"></span>
3110 - </button>
3111 - <?php endif; ?>
3112 - </div>
3113 - <div class="mxchat-content-full" style="display: none;">
3114 - <div class="content-view">
3115 - <?php
3116 - if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
3117 - echo '<div dir="rtl" lang="he" class="rtl-content">';
3118 - echo wp_kses_post(wpautop($content));
3119 - echo '</div>';
3120 - } else {
3121 - echo wp_kses_post(wpautop($content));
3122 - }
3123 - ?>
3124 - </div>
3125 - </div>
3126 - </div>
3127 - </td>
3128 - <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3129 - <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
3130 - </td>
3131 - <td class="mxchat-actions-cell" style="padding: 12px 16px;">
3132 - <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
3133 - </td>
3134 - </tr>
3135 - <?php
3136 - }
3137 - } else {
3138 - // Single entry - display normally with accordion
3139 - $prompt = $first_prompt;
3140 - $content = isset($prompt->display_content) ? $prompt->display_content : $prompt->article_content;
3141 - $content_preview = mb_strlen($content) > $preview_length
3142 - ? mb_substr($content, 0, $preview_length) . '...'
3143 - : $content;
3144 - ?>
3145 - <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
3146 - data-source="<?php echo esc_attr($data_source); ?>"
3147 - style="border-bottom: 1px solid var(--mxch-card-border);">
3148 - <td style="padding: 12px 16px; text-align: center;">
3149 - <input type="checkbox"
3150 - class="mxchat-entry-checkbox"
3151 - data-entry-id="<?php echo esc_attr($prompt->id); ?>"
3152 - data-source="<?php echo esc_attr($data_source); ?>"
3153 - data-source-url="<?php echo esc_attr($source_url); ?>"
3154 - data-is-group="false">
3155 - </td>
3156 - <td style="padding: 12px 16px; font-size: 13px;">
3157 - <?php echo esc_html($prompt->id); ?>
3158 - </td>
3159 - <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3160 - <div class="mxchat-accordion-wrapper">
3161 - <div class="mxchat-content-preview">
3162 - <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
3163 - <?php if (mb_strlen($content) > $preview_length) : ?>
3164 - <button class="mxchat-expand-toggle" type="button">
3165 - <span class="dashicons dashicons-arrow-down-alt2"></span>
3166 - </button>
3167 - <?php endif; ?>
3168 - </div>
3169 - <div class="mxchat-content-full" style="display: none;">
3170 - <div class="content-view">
3171 - <?php
3172 - if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
3173 - echo '<div dir="rtl" lang="he" class="rtl-content">';
3174 - echo wp_kses_post(wpautop($content));
3175 - echo '</div>';
3176 - } else {
3177 - echo wp_kses_post(wpautop($content));
3178 - }
3179 - ?>
3180 - </div>
3181 - </div>
3182 - </div>
3183 - </td>
3184 - <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3185 - <?php
3186 - $actual_source = $source_url;
3187 - if (strpos($source_url, '_ungrouped_') === 0) {
3188 - $actual_source = $prompt->source_url ?? '';
3189 - }
3190 - if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
3191 - <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
3192 - <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
3193 - <?php esc_html_e('View', 'mxchat'); ?>
3194 - </a>
3195 - <?php else : ?>
3196 - <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
3197 - <?php endif; ?>
3198 - </td>
3199 - <td style="padding: 12px 16px; white-space: nowrap;">
3200 - <button type="button"
3201 - class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
3202 - data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
3203 - data-entry-id="<?php echo esc_attr($prompt->id); ?>"
3204 - data-data-source="<?php echo esc_attr($data_source); ?>"
3205 - data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3206 - data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
3207 - title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
3208 - <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
3209 - </button>
3210 - <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-wordpress" data-entry-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_wordpress_prompt_nonce'); ?>" style="color: var(--mxch-error);">
3211 - <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
3212 - </button>
3213 - </td>
3214 - </tr>
3215 - <?php
3216 - }
3217 - }
3218 - }
3219 - $html = ob_get_clean();
3220 -
3221 - // Generate pagination HTML (include search/filter data for subsequent pages)
3222 - $pagination_html = '';
3223 - if ($total_pages > 1) {
3224 - $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
3225 -
3226 - // Previous button
3227 - if ($page > 1) {
3228 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
3229 - }
3230 -
3231 - // Page numbers
3232 - $start_page = max(1, $page - 2);
3233 - $end_page = min($total_pages, $page + 2);
3234 -
3235 - if ($start_page > 1) {
3236 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
3237 - if ($start_page > 2) {
3238 - $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
3239 - }
3240 - }
3241 -
3242 - for ($i = $start_page; $i <= $end_page; $i++) {
3243 - if ($i == $page) {
3244 - $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
3245 - } else {
3246 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
3247 - }
3248 - }
3249 -
3250 - if ($end_page < $total_pages) {
3251 - if ($end_page < $total_pages - 1) {
3252 - $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
3253 - }
3254 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
3255 - }
3256 -
3257 - // Next button
3258 - if ($page < $total_pages) {
3259 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
3260 - }
3261 -
3262 - $pagination_html .= '</div>';
3263 - }
3264 -
3265 - wp_send_json_success(array(
3266 - 'html' => $html,
3267 - 'pagination_html' => $pagination_html,
3268 - 'total_count' => $total_records,
3269 - 'total_pages' => $total_pages,
3270 - 'page' => $page,
3271 - 'per_page' => $per_page,
3272 - 'data_source' => 'wordpress'
3273 - ));
3274 -}
3275 -
3276 -/**
3277 - * AJAX handler to detect available sitemaps on the site
3278 - * Optimized for speed - only checks primary sitemap indexes first
3279 - */
3280 -public function ajax_mxchat_detect_sitemaps() {
3281 - check_ajax_referer('mxchat_detect_sitemaps_nonce', 'nonce');
3282 -
3283 - if (!current_user_can('manage_options')) {
3284 - wp_send_json_error(array('message' => 'Unauthorized'));
3285 - return;
3286 - }
3287 -
3288 - $site_url = get_site_url();
3289 - $sitemaps = array();
3290 - $found_index = false;
3291 -
3292 - // Only check the main sitemap index files first (much faster)
3293 - // These are the primary entry points that contain sub-sitemaps
3294 - $primary_indexes = array(
3295 - 'sitemap_index.xml' => 'Yoast SEO / Rank Math', // Yoast & Rank Math
3296 - 'wp-sitemap.xml' => 'WordPress Core', // WordPress Core
3297 - 'sitemap.xml' => 'Standard', // Generic/AIOSEO
3298 - );
3299 -
3300 - foreach ($primary_indexes as $path => $source) {
3301 - $url = trailingslashit($site_url) . $path;
3302 -
3303 - $response = wp_remote_head($url, array(
3304 - 'timeout' => 10,
3305 - 'sslverify' => false,
3306 - 'redirection' => 1,
3307 - 'user-agent' => mxchat_ingest_user_agent(),
3308 - ));
3309 -
3310 - if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3311 - // Found a sitemap index - parse it to get sub-sitemaps
3312 - $sub_sitemaps = $this->parse_sitemap_index($url);
3313 - if (!empty($sub_sitemaps)) {
3314 - $sitemaps[] = array(
3315 - 'url' => $url,
3316 - 'type' => 'index',
3317 - 'source' => $source,
3318 - 'sub_sitemaps' => $sub_sitemaps
3319 - );
3320 - $found_index = true;
3321 - // Found a valid index, no need to check others
3322 - break;
3323 - }
3324 - }
3325 - }
3326 -
3327 - // If no sitemap index found, check for standalone sitemaps
3328 - if (!$found_index) {
3329 - $standalone_sitemaps = array(
3330 - 'post-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
3331 - 'page-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
3332 - );
3333 -
3334 - foreach ($standalone_sitemaps as $path => $info) {
3335 - $url = trailingslashit($site_url) . $path;
3336 -
3337 - $response = wp_remote_head($url, array(
3338 - 'timeout' => 2,
3339 - 'sslverify' => false
3340 - ));
3341 -
3342 - if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3343 - $sitemaps[] = array(
3344 - 'url' => $url,
3345 - 'type' => $info['type'],
3346 - 'source' => $info['source'],
3347 - 'url_count' => 0 // Skip URL count for speed
3348 - );
3349 - }
3350 - }
3351 - }
3352 -
3353 - wp_send_json_success(array(
3354 - 'sitemaps' => $sitemaps,
3355 - 'site_url' => $site_url
3356 - ));
3357 -}
3358 -
3359 -/**
3360 - * Parse a sitemap index to get sub-sitemaps
3361 - * Optimized: doesn't fetch URL count for each sub-sitemap (too slow)
3362 - */
3363 -private function parse_sitemap_index($url) {
3364 - $sub_sitemaps = array();
3365 -
3366 - $response = wp_remote_get($url, array(
3367 - 'timeout' => 30,
3368 - 'sslverify' => false,
3369 - 'user-agent' => mxchat_ingest_user_agent(),
3370 - 'headers' => array(
3371 - 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
3372 - ),
3373 - ));
3374 -
3375 - if (is_wp_error($response)) {
3376 - return $sub_sitemaps;
3377 - }
3378 -
3379 - $body = wp_remote_retrieve_body($response);
3380 - if (empty($body)) {
3381 - return $sub_sitemaps;
3382 - }
3383 -
3384 - // Suppress XML errors
3385 - libxml_use_internal_errors(true);
3386 - $xml = simplexml_load_string($body);
3387 - libxml_clear_errors();
3388 -
3389 - if ($xml === false) {
3390 - return $sub_sitemaps;
3391 - }
3392 -
3393 - // Check if it's a sitemap index (contains <sitemap> elements)
3394 - if (isset($xml->sitemap)) {
3395 - foreach ($xml->sitemap as $sitemap) {
3396 - $loc = (string) $sitemap->loc;
3397 - if (!empty($loc)) {
3398 - // Try to determine the type from the URL
3399 - $type = 'content';
3400 - if (strpos($loc, 'category') !== false || strpos($loc, 'tag') !== false || strpos($loc, 'taxonomy') !== false) {
3401 - $type = 'taxonomy';
3402 - } elseif (strpos($loc, 'author') !== false || strpos($loc, 'user') !== false) {
3403 - $type = 'author';
3404 - }
3405 -
3406 - // Skip URL count - too slow to fetch for each sitemap
3407 - $sub_sitemaps[] = array(
3408 - 'url' => $loc,
3409 - 'type' => $type,
3410 - 'url_count' => 0, // Don't fetch - takes too long
3411 - 'name' => basename(parse_url($loc, PHP_URL_PATH))
3412 - );
3413 - }
3414 - }
3415 - }
3416 -
3417 - return $sub_sitemaps;
3418 -}
3419 -
3420 -/**
3421 - * Get URL count from a sitemap
3422 - */
3423 -private function get_sitemap_url_count($url) {
3424 - $response = wp_remote_get($url, array(
3425 - 'timeout' => 30,
3426 - 'sslverify' => false,
3427 - 'user-agent' => mxchat_ingest_user_agent(),
3428 - 'headers' => array(
3429 - 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
3430 - ),
3431 - ));
3432 -
3433 - if (is_wp_error($response)) {
3434 - return 0;
3435 - }
3436 -
3437 - $body = wp_remote_retrieve_body($response);
3438 - if (empty($body)) {
3439 - return 0;
3440 - }
3441 -
3442 - // Count <url> or <loc> elements
3443 - $count = preg_match_all('/<url>/i', $body, $matches);
3444 - return $count ?: 0;
3445 -}
3446 -
3447 -/**
3448 - * Get sitemaps declared in robots.txt
3449 - */
3450 -private function get_sitemaps_from_robots($site_url) {
3451 - $sitemaps = array();
3452 - $robots_url = trailingslashit($site_url) . 'robots.txt';
3453 -
3454 - $response = wp_remote_get($robots_url, array(
3455 - 'timeout' => 15,
3456 - 'sslverify' => false,
3457 - 'user-agent' => mxchat_ingest_user_agent(),
3458 - ));
3459 -
3460 - if (is_wp_error($response)) {
3461 - return $sitemaps;
3462 - }
3463 -
3464 - $body = wp_remote_retrieve_body($response);
3465 - if (empty($body)) {
3466 - return $sitemaps;
3467 - }
3468 -
3469 - // Find Sitemap: declarations
3470 - if (preg_match_all('/^Sitemap:\s*(.+)$/mi', $body, $matches)) {
3471 - foreach ($matches[1] as $sitemap_url) {
3472 - $sitemap_url = trim($sitemap_url);
3473 - if (filter_var($sitemap_url, FILTER_VALIDATE_URL)) {
3474 - $sitemaps[] = $sitemap_url;
3475 - }
3476 - }
3477 - }
3478 -
3479 - return $sitemaps;
3480 -}
3481 -
3482 1767 public function mxchat_stop_processing() {
3483 1768 // Verify permissions
3484 1769 if (!current_user_can('manage_options')) {
3485 1770 wp_die(esc_html__('Unauthorized access', 'mxchat'));
@@ -3487,52 +1772,28 @@
3487 1772
3488 1773 // Verify nonce
3489 1774 check_admin_referer('mxchat_stop_processing_action', 'mxchat_stop_processing_nonce');
3490 1775
3491 - global $wpdb;
3492 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
3493 -
3494 - // Get active queue IDs
3495 - $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
3496 - $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
3497 -
3498 - // Delete all pending items from active queues
3499 - if ($sitemap_queue_id) {
3500 - $wpdb->delete(
3501 - $table_name,
3502 - array(
3503 - 'queue_id' => $sitemap_queue_id,
3504 - 'status' => 'pending'
3505 - ),
3506 - array('%s', '%s')
3507 - );
3508 -
3509 - delete_transient('mxchat_active_queue_sitemap');
1776 + // Get the last sitemap URL and clear its transient
1777 + $sitemap_url = get_transient('mxchat_last_sitemap_url');
1778 + if ($sitemap_url) {
1779 + delete_transient('mxchat_sitemap_status_' . md5($sitemap_url));
3510 1780 delete_transient('mxchat_last_sitemap_url');
3511 1781 }
3512 -
3513 - if ($pdf_queue_id) {
3514 - // Get PDF path before deleting
3515 - $pdf_path = $this->mxchat_get_queue_meta($pdf_queue_id, 'pdf_path');
3516 -
3517 - $wpdb->delete(
3518 - $table_name,
3519 - array(
3520 - 'queue_id' => $pdf_queue_id,
3521 - 'status' => 'pending'
3522 - ),
3523 - array('%s', '%s')
3524 - );
3525 -
3526 - // Delete PDF file
3527 - if ($pdf_path && file_exists($pdf_path)) {
3528 - wp_delete_file($pdf_path);
3529 - }
3530 -
3531 - delete_transient('mxchat_active_queue_pdf');
1782 +
1783 + // Get the last PDF URL and clear its transient
1784 + $pdf_url = get_transient('mxchat_last_pdf_url');
1785 + if ($pdf_url) {
1786 + delete_transient('mxchat_pdf_status_' . md5($pdf_url));
3532 1787 delete_transient('mxchat_last_pdf_url');
3533 1788 }
3534 1789
1790 + // Unschedule any pending sitemap events
1791 + $timestamp = wp_next_scheduled('mxchat_process_sitemap_urls');
1792 + if ($timestamp) {
1793 + wp_unschedule_event($timestamp, 'mxchat_process_sitemap_urls');
1794 + }
1795 +
3535 1796 // Redirect back with a success message
3536 1797 set_transient('mxchat_admin_notice_success',
3537 1798 esc_html__('Processing has been stopped successfully.', 'mxchat'),
3538 1799 30
@@ -3539,12 +1800,8 @@
3539 1800 );
3540 1801 wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
3541 1802 exit;
3542 1803 }
3543 -
3544 -/**
3545 - * Get content list for processing
3546 - */
3547 1804 public function ajax_mxchat_get_content_list() {
3548 1805 // Verify the nonce
3549 1806 check_ajax_referer('mxchat_content_selector_nonce', 'nonce');
3550 1807
@@ -3552,9 +1809,9 @@
3552 1809 wp_send_json_error(__('Unauthorized access', 'mxchat'));
3553 1810 }
3554 1811
3555 1812 $page = isset($_GET['page']) ? absint($_GET['page']) : 1;
3556 - $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 100;
1813 + $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 20;
3557 1814 $search = isset($_GET['search']) ? sanitize_text_field($_GET['search']) : '';
3558 1815 $post_type = isset($_GET['post_type']) ? sanitize_text_field($_GET['post_type']) : 'all';
3559 1816 $post_status = isset($_GET['post_status']) ? sanitize_text_field($_GET['post_status']) : 'publish';
3560 1817 $processed_filter = isset($_GET['processed_filter']) ? sanitize_text_field($_GET['processed_filter']) : 'all';
@@ -3567,47 +1824,20 @@
3567 1824 'orderby' => 'date',
3568 1825 'order' => 'DESC',
3569 1826 );
3570 1827
3571 - // Handle post types - IMPROVED VERSION
1828 + // Handle post types
3572 1829 if ($post_type !== 'all') {
3573 1830 $args['post_type'] = $post_type;
3574 1831 } else {
3575 - // Get all available post types that might contain content
3576 - $all_post_types = array();
1832 + // Default to post and page if we can't get post types
1833 + $args['post_type'] = array('post', 'page');
3577 1834
3578 - // First get all public post types
3579 - $public_types = get_post_types(array('public' => true), 'names');
3580 - $all_post_types = array_merge($all_post_types, $public_types);
3581 -
3582 - // Add common forum/community post types
3583 - $forum_types = array('topic', 'reply', 'forum', 'wpforo_topic', 'wpforo_post');
3584 - foreach ($forum_types as $forum_type) {
3585 - if (post_type_exists($forum_type)) {
3586 - $all_post_types[] = $forum_type;
3587 - }
1835 + // Try to get public post types
1836 + $public_types = $this->mxchat_get_public_post_types();
1837 + if (is_array($public_types) && !empty($public_types)) {
1838 + $args['post_type'] = array_keys($public_types);
3588 1839 }
3589 -
3590 - // Add other commonly used post types
3591 - $common_types = array('product', 'job_listing', 'event', 'portfolio');
3592 - foreach ($common_types as $common_type) {
3593 - if (post_type_exists($common_type)) {
3594 - $all_post_types[] = $common_type;
3595 - }
3596 - }
3597 -
3598 - // Remove duplicates and ensure we have at least some post types
3599 - $all_post_types = array_unique($all_post_types);
3600 -
3601 - if (empty($all_post_types)) {
3602 - // Fallback to basic post types
3603 - $all_post_types = array('post', 'page');
3604 - }
3605 -
3606 - $args['post_type'] = $all_post_types;
3607 -
3608 - // Debug logging to see what post types are being queried
3609 - //error_log('MxChat Debug: Querying post types: ' . implode(', ', $all_post_types));
3610 1840 }
3611 1841
3612 1842 if (!empty($search)) {
3613 1843 $args['s'] = $search;
@@ -3612,58 +1842,36 @@
3612 1842 if (!empty($search)) {
3613 1843 $args['s'] = $search;
3614 1844 }
3615 1845
3616 - // Get processed data from storage
1846 + // ================================
1847 + // FIXED: Check only the ACTIVE storage method
1848 + // ================================
1849 +
3617 1850 $processed_data = array();
3618 1851
1852 + $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
3619 1853 $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
3620 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3621 -
1854 + $pinecone_manager->mxchat_refresh_after_new_content($pinecone_options);
1855 +
3622 1856 if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3623 - // Get fresh data from Pinecone - no caching
1857 + // ONLY check Pinecone if it's enabled
3624 1858 $processed_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
3625 1859 } else {
3626 - // WordPress DB checking with better URL matching for all post types
1860 + // ONLY check WordPress DB if Pinecone is not enabled
3627 1861 global $wpdb;
3628 1862 $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3629 - $processed_items = $wpdb->get_results("SELECT id, source_url, article_content, timestamp FROM {$table_name}");
3630 -
3631 - // Group items by source_url to count chunks
3632 - $url_chunk_counts = array();
3633 - $url_latest_timestamp = array();
3634 - $url_first_id = array();
3635 -
1863 + $processed_items = $wpdb->get_results("SELECT id, source_url, timestamp FROM {$table_name}");
1864 +
3636 1865 if (!empty($processed_items)) {
3637 1866 foreach ($processed_items as $item) {
3638 - $url = $item->source_url;
3639 - if (empty($url)) continue;
3640 -
3641 - // Count chunks per URL
3642 - if (!isset($url_chunk_counts[$url])) {
3643 - $url_chunk_counts[$url] = 0;
3644 - $url_latest_timestamp[$url] = $item->timestamp;
3645 - $url_first_id[$url] = $item->id;
3646 - }
3647 - $url_chunk_counts[$url]++;
3648 -
3649 - // Track latest timestamp
3650 - if (strtotime($item->timestamp) > strtotime($url_latest_timestamp[$url])) {
3651 - $url_latest_timestamp[$url] = $item->timestamp;
3652 - }
3653 - }
3654 -
3655 - // Now build processed_data with chunk counts
3656 - foreach ($url_chunk_counts as $url => $chunk_count) {
3657 - $post_id = $this->mxchat_url_to_post_id_improved($url);
3658 -
1867 + $post_id = url_to_postid($item->source_url);
3659 1868 if ($post_id) {
3660 1869 $processed_data[$post_id] = array(
3661 - 'db_id' => $url_first_id[$url],
3662 - 'timestamp' => $url_latest_timestamp[$url],
3663 - 'url' => $url,
3664 - 'source' => 'wordpress',
3665 - 'chunk_count' => $chunk_count
1870 + 'db_id' => $item->id,
1871 + 'timestamp' => $item->timestamp,
1872 + 'url' => $item->source_url,
1873 + 'source' => 'wordpress'
3666 1874 );
3667 1875 }
3668 1876 }
3669 1877 }
@@ -3668,8 +1876,10 @@
3668 1876 }
3669 1877 }
3670 1878 }
3671 1879
1880 + // ================================
1881 +
3672 1882 // Get processed IDs as a simple array for in_array checks
3673 1883 $processed_ids = array_keys($processed_data);
3674 1884
3675 1885 // Handle processed/unprocessed filter
@@ -3711,14 +1921,8 @@
3711 1921 $db_record_id = $item_data['db_id'];
3712 1922 }
3713 1923 }
3714 1924
3715 - // Get chunk count for this item
3716 - $chunk_count = 0;
3717 - if ($is_processed && isset($processed_data[$id]['chunk_count'])) {
3718 - $chunk_count = intval($processed_data[$id]['chunk_count']);
3719 - }
3720 -
3721 1925 $content_items[] = array(
3722 1926 'id' => $id,
3723 1927 'title' => get_the_title(),
3724 1928 'permalink' => get_permalink(),
@@ -3729,10 +1933,9 @@
3729 1933 'word_count' => $word_count,
3730 1934 'already_processed' => $is_processed,
3731 1935 'processed_date' => $processed_date,
3732 1936 'db_record_id' => $db_record_id,
3733 - 'data_source' => $data_source,
3734 - 'chunk_count' => $chunk_count
1937 + 'data_source' => $data_source
3735 1938 );
3736 1939 }
3737 1940 wp_reset_postdata();
3738 1941 }
@@ -3748,178 +1951,8 @@
3748 1951 wp_send_json_success($response);
3749 1952 exit;
3750 1953 }
3751 1954
3752 -
3753 -/**
3754 - * This function handles various WooCommerce URL formats and permalink structures
3755 - */
3756 -private function mxchat_url_to_post_id_improved($url) {
3757 - // First try the standard WordPress function
3758 - $post_id = url_to_postid($url);
3759 -
3760 - if ($post_id > 0) {
3761 - return $post_id;
3762 - }
3763 -
3764 - // If that fails, try more aggressive URL matching
3765 - // Remove trailing slashes and query parameters for better matching
3766 - $clean_url = rtrim($url, '/');
3767 - $clean_url = strtok($clean_url, '?'); // Remove query parameters
3768 -
3769 - // Try again with cleaned URL
3770 - $post_id = url_to_postid($clean_url);
3771 - if ($post_id > 0) {
3772 - return $post_id;
3773 - }
3774 -
3775 - // For bbPress forum topics, try extracting slug from URL
3776 - if (strpos($url, '/topic/') !== false || strpos($url, '/forums/') !== false) {
3777 - // Handle bbPress URLs: /forums/topic/topic-name/
3778 - if (preg_match('/\/forums\/topic\/([^\/\?]+)/', $url, $matches)) {
3779 - $topic_slug = $matches[1];
3780 -
3781 - // Look up topic by slug
3782 - $topic = get_page_by_path($topic_slug, OBJECT, 'topic');
3783 - if ($topic) {
3784 - return $topic->ID;
3785 - }
3786 -
3787 - // Alternative method: query by post_name
3788 - global $wpdb;
3789 - $post_id = $wpdb->get_var($wpdb->prepare(
3790 - "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3791 - $topic_slug
3792 - ));
3793 -
3794 - if ($post_id) {
3795 - return intval($post_id);
3796 - }
3797 - }
3798 -
3799 - // Handle simpler topic URLs: /topic/topic-name/
3800 - if (preg_match('/\/topic\/([^\/\?]+)/', $url, $matches)) {
3801 - $topic_slug = $matches[1];
3802 -
3803 - global $wpdb;
3804 - $post_id = $wpdb->get_var($wpdb->prepare(
3805 - "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3806 - $topic_slug
3807 - ));
3808 -
3809 - if ($post_id) {
3810 - return intval($post_id);
3811 - }
3812 - }
3813 - }
3814 -
3815 - // For WooCommerce products
3816 - if (strpos($url, '/product/') !== false || strpos($url, 'product=') !== false) {
3817 - // Extract product slug from various URL formats
3818 - $product_slug = '';
3819 -
3820 - // Handle pretty permalinks: /product/product-name/
3821 - if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
3822 - $product_slug = $matches[1];
3823 - }
3824 - // Handle query parameters: ?product=product-name
3825 - elseif (preg_match('/[\?&]product=([^&]+)/', $url, $matches)) {
3826 - $product_slug = $matches[1];
3827 - }
3828 -
3829 - if (!empty($product_slug)) {
3830 - // Look up product by slug
3831 - $product = get_page_by_path($product_slug, OBJECT, 'product');
3832 - if ($product) {
3833 - return $product->ID;
3834 - }
3835 -
3836 - // Alternative method: query by post_name
3837 - global $wpdb;
3838 - $post_id = $wpdb->get_var($wpdb->prepare(
3839 - "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'product' AND post_status = 'publish'",
3840 - $product_slug
3841 - ));
3842 -
3843 - if ($post_id) {
3844 - return intval($post_id);
3845 - }
3846 - }
3847 - }
3848 -
3849 - // Generic approach: try to extract slug and match against all post types
3850 - $parsed_url = wp_parse_url($clean_url);
3851 - $path = $parsed_url['path'] ?? '';
3852 -
3853 - if (!empty($path)) {
3854 - // Get the last part of the path as potential slug
3855 - $path_parts = array_filter(explode('/', trim($path, '/')));
3856 - $potential_slug = end($path_parts);
3857 -
3858 - if (!empty($potential_slug)) {
3859 - global $wpdb;
3860 -
3861 - // Try to find any post with this slug
3862 - $post_id = $wpdb->get_var($wpdb->prepare(
3863 - "SELECT ID FROM {$wpdb->posts}
3864 - WHERE post_name = %s
3865 - AND post_status IN ('publish', 'closed', 'private')
3866 - AND post_type NOT IN ('revision', 'attachment', 'nav_menu_item')
3867 - ORDER BY CASE
3868 - WHEN post_type = 'post' THEN 1
3869 - WHEN post_type = 'page' THEN 2
3870 - WHEN post_type = 'topic' THEN 3
3871 - WHEN post_type = 'product' THEN 4
3872 - ELSE 5
3873 - END
3874 - LIMIT 1",
3875 - $potential_slug
3876 - ));
3877 -
3878 - if ($post_id) {
3879 - return intval($post_id);
3880 - }
3881 - }
3882 - }
3883 -
3884 - // ADDITIONAL: Try direct database lookup by URL variations
3885 - global $wpdb;
3886 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3887 -
3888 - // Try variations of the URL (with/without trailing slash, http/https)
3889 - $url_variations = array(
3890 - $url,
3891 - rtrim($url, '/'),
3892 - $url . '/',
3893 - str_replace('http://', 'https://', $url),
3894 - str_replace('https://', 'http://', $url),
3895 - str_replace('http://', 'https://', rtrim($url, '/')),
3896 - str_replace('https://', 'http://', rtrim($url, '/'))
3897 - );
3898 -
3899 - // Remove duplicates
3900 - $url_variations = array_unique($url_variations);
3901 -
3902 - foreach ($url_variations as $variation) {
3903 - $existing_record = $wpdb->get_row($wpdb->prepare(
3904 - "SELECT id, source_url FROM $table_name WHERE source_url = %s",
3905 - $variation
3906 - ));
3907 -
3908 - if ($existing_record) {
3909 - // Try to get post ID from this stored URL
3910 - $stored_post_id = url_to_postid($existing_record->source_url);
3911 - if ($stored_post_id > 0) {
3912 - return $stored_post_id;
3913 - }
3914 - }
3915 - }
3916 -
3917 - return 0; // No match found
3918 -}
3919 -/**
3920 - * Process selected content via AJAX
3921 - */
3922 1955 public function ajax_mxchat_process_selected_content() {
3923 1956 // Basic request validation
3924 1957 if (!check_ajax_referer('mxchat_content_selector_nonce', 'nonce', false)) {
3925 1958 wp_send_json_error('Invalid nonce');
@@ -3943,16 +1976,8 @@
3943 1976 wp_send_json_error('No content selected');
3944 1977 exit;
3945 1978 }
3946 1979
3947 - // Get bot_id from request
3948 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
3949 -
3950 - // ACF→PDF extraction is an install-level setting (Knowledge → ACF Fields,
3951 - // plan 11720c). The import modal shows a passive status line pointing
3952 - // there; the old per-batch checkbox and its remembered default are gone.
3953 - $extract_acf_pdfs = get_option('mxchat_acf_pdf_extraction', '0') === '1';
3954 -
3955 1980 // Process only ONE post at a time to avoid request size issues
3956 1981 $post_id = reset($post_ids);
3957 1982 $post = get_post($post_id);
3958 1983
@@ -3959,224 +1984,54 @@
3959 1984 if (!$post) {
3960 1985 wp_send_json_error('Post not found');
3961 1986 exit;
3962 1987 }
3963 -
3964 - /**
3965 - * Allow developers to modify post data before processing into the knowledge base.
3966 - * Applied on BOTH content-preparation paths (this manual bulk import and the
3967 - * auto-sync path in mxchat_handle_post_update) with the same signature, so a
3968 - * callback registered once covers every indexing route. Purely additive —
3969 - * zero behaviour change when unhooked.
3970 - *
3971 - * @param WP_Post $post The post about to be indexed.
3972 - * @param string $bot_id Bot context for this import.
3973 - */
3974 - $post = apply_filters('mxchat_before_process_post', $post, $bot_id);
3975 - if (!($post instanceof WP_Post)) {
3976 - $post = get_post($post_id); // defend against a bad callback return
1988 +
1989 + // Get minimal content
1990 + $content = $post->post_title . "\n\n" . wp_strip_all_tags($post->post_content);
1991 + $content = substr($content, 0, 10000); // Limit content size
1992 +
1993 + // Get API key with proper model detection
1994 + $options = get_option('mxchat_options');
1995 + $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
1996 +
1997 + if (strpos($selected_model, 'voyage') === 0) {
1998 + $api_key = $options['voyage_api_key'] ?? '';
1999 + $provider_name = 'Voyage AI';
2000 + } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
2001 + $api_key = $options['gemini_api_key'] ?? '';
2002 + $provider_name = 'Google Gemini';
2003 + } else {
2004 + $api_key = $options['api_key'] ?? '';
2005 + $provider_name = 'OpenAI';
3977 2006 }
3978 -
3979 - // Get content including title, short description (for WooCommerce), and main content
3980 - $content = $post->post_title . "\n\n";
3981 -
3982 - // Add short description if it exists (WooCommerce products use post_excerpt for short description)
3983 - if (!empty($post->post_excerpt)) {
3984 - // Remove shortcode tags but preserve content inside them
3985 - $clean_excerpt = $this->strip_shortcode_tags_preserve_content($post->post_excerpt);
3986 - $content .= "Short Description: " . wp_strip_all_tags($clean_excerpt) . "\n\n";
3987 - }
3988 -
3989 - // Add main content - remove shortcode tags but preserve content inside them
3990 - $clean_content = $this->strip_shortcode_tags_preserve_content($post->post_content);
3991 - $content .= wp_strip_all_tags($clean_content);
3992 -
3993 - // ADD WOOCOMMERCE PRODUCT DATA (pricing, stock, categories, custom tabs)
3994 - if (get_post_type($post_id) === 'product' && class_exists('WooCommerce')) {
3995 - $product = wc_get_product($post_id);
3996 -
3997 - if ($product) {
3998 - // Get pricing information
3999 - $regular_price = $product->get_regular_price();
4000 - $sale_price = $product->get_sale_price();
4001 - $price = $product->get_price();
4002 - $sku = $product->get_sku();
4003 -
4004 - // Get currency symbol
4005 - $currency_symbol = get_woocommerce_currency_symbol();
4006 -
4007 - // Add pricing information
4008 - $content .= "\n";
4009 - if (!empty($regular_price)) {
4010 - $content .= "Price: " . $currency_symbol . $regular_price . "\n";
4011 - } elseif (!empty($price)) {
4012 - $content .= "Price: " . $currency_symbol . $price . "\n";
4013 - }
4014 -
4015 - if (!empty($sale_price) && $sale_price !== $regular_price) {
4016 - $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
4017 - }
4018 -
4019 - // Handle variable products - show price range
4020 - if ($product->is_type('variable')) {
4021 - $min_price = $product->get_variation_price('min');
4022 - $max_price = $product->get_variation_price('max');
4023 - if ($min_price !== $max_price) {
4024 - $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
4025 - }
4026 - }
4027 -
4028 - if (!empty($sku)) {
4029 - $content .= "SKU: " . $sku . "\n";
4030 - }
4031 -
4032 - // Get product categories
4033 - $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
4034 - if (!empty($categories) && !is_wp_error($categories)) {
4035 - $content .= "Categories: " . implode(', ', $categories) . "\n";
4036 - }
4037 - }
4038 -
4039 - // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
4040 - $custom_tabs = get_post_meta($post_id, 'yikes_woo_products_tabs', true);
4041 - if (!empty($custom_tabs) && is_array($custom_tabs)) {
4042 - foreach ($custom_tabs as $tab) {
4043 - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
4044 - $tab_content = isset($tab['content']) ? $tab['content'] : '';
4045 -
4046 - if (!empty($tab_title) && !empty($tab_content)) {
4047 - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
4048 - }
4049 - }
4050 - }
4051 -
4052 - // Also check for reusable/saved tabs applied to this product
4053 - $applied_saved_tabs = get_post_meta($post_id, 'yikes_woo_reusable_products_tabs_applied', true);
4054 - if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
4055 - $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
4056 - if (!empty($saved_tabs) && is_array($saved_tabs)) {
4057 - foreach ($applied_saved_tabs as $saved_tab_id) {
4058 - if (isset($saved_tabs[$saved_tab_id])) {
4059 - $tab = $saved_tabs[$saved_tab_id];
4060 - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
4061 - $tab_content = isset($tab['content']) ? $tab['content'] : '';
4062 -
4063 - if (!empty($tab_title) && !empty($tab_content)) {
4064 - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
4065 - }
4066 - }
4067 - }
4068 - }
4069 - }
4070 - }
4071 -
4072 - // ADD ACF FIELDS SUPPORT
4073 - $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
4074 - $pdf_extracted_count = 0;
4075 - if (!empty($acf_fields)) {
4076 - $acf_content_parts = array();
4077 - $pdf_attachment_ids = array();
4078 -
4079 - foreach ($acf_fields as $field_name => $field_value) {
4080 - $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
4081 -
4082 - if (!empty($formatted_value)) {
4083 - $field_label = ucwords(str_replace('_', ' ', $field_name));
4084 - $acf_content_parts[] = $field_label . ": " . $formatted_value;
4085 - }
4086 -
4087 - // Walk this field's value tree for any PDF attachment references and queue them for extraction.
4088 - // Only when the user opted into ACF→PDF extraction for this batch; otherwise the ACF text
4089 - // still lands in the KB but the heavier PDF parsing is skipped.
4090 - if ($extract_acf_pdfs) {
4091 - $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
4092 - }
4093 - }
4094 -
4095 - if (!empty($acf_content_parts)) {
4096 - $content .= "\n\n" . implode("\n", $acf_content_parts);
4097 - }
4098 -
4099 - // Extract text from each unique PDF found in ACF fields and append as a labeled section
4100 - if ($extract_acf_pdfs && !empty($pdf_attachment_ids)) {
4101 - $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
4102 - $pdf_sections = array();
4103 - foreach ($pdf_attachment_ids as $att_id) {
4104 - $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
4105 - if (!empty($pdf_text)) {
4106 - $pdf_title = get_the_title($att_id);
4107 - $pdf_url = wp_get_attachment_url($att_id);
4108 - $header = 'PDF Attachment';
4109 - if (!empty($pdf_title)) {
4110 - $header .= ': ' . $pdf_title;
4111 - }
4112 - if (!empty($pdf_url)) {
4113 - $header .= ' (' . $pdf_url . ')';
4114 - }
4115 - $pdf_sections[] = $header . "\n" . $pdf_text;
4116 - $pdf_extracted_count++;
4117 - }
4118 - }
4119 - if (!empty($pdf_sections)) {
4120 - $content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
4121 - }
4122 - }
4123 - }
4124 -
4125 - // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
4126 - $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
4127 - if (!empty($custom_meta)) {
4128 - $meta_content_parts = array();
4129 -
4130 - foreach ($custom_meta as $meta_key => $meta_value) {
4131 - // Convert meta key to readable label
4132 - $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
4133 - $meta_content_parts[] = $meta_label . ": " . $meta_value;
4134 - }
4135 -
4136 - if (!empty($meta_content_parts)) {
4137 - $content .= "\n\n" . implode("\n", $meta_content_parts);
4138 - }
4139 - }
4140 -
4141 - // Debug logging for WordPress Import content
4142 - //error_log('[MXCHAT-WP-IMPORT-DEBUG] Post ID: ' . $post_id . ' Title: ' . $post->post_title);
4143 - //error_log('[MXCHAT-WP-IMPORT-DEBUG] Raw post_content length: ' . strlen($post->post_content));
4144 - //error_log('[MXCHAT-WP-IMPORT-DEBUG] Final content length: ' . strlen($content));
4145 - //error_log('[MXCHAT-WP-IMPORT-DEBUG] Content preview: ' . substr($content, 0, 300));
4146 -
4147 - // Note: Removed 10,000 char limit - chunking now handles large content properly
4148 -
4149 - // Get bot-specific embedding decision — custom-provider-aware (plan cbd5fd)
4150 - $bot_options = $this->get_bot_options($bot_id);
4151 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
4152 -
4153 - $preflight = MxChat_Utils::embedding_preflight($options);
4154 - if (!$preflight['ok']) {
4155 - MxChat_Admin::mxchat_log_debug('api_error', $preflight['reason'] . ' (knowledge processing)');
4156 - wp_send_json_error($preflight['reason']);
2007 +
2008 + if (empty($api_key)) {
2009 + wp_send_json_error($provider_name . ' API key not configured');
4157 2010 exit;
4158 2011 }
4159 - $api_key = $preflight['api_key'];
4160 2012
4161 2013 $source_url = get_permalink($post_id);
4162 2014 $vector_id = md5($source_url); // Vector ID for Pinecone
4163 2015
4164 - // Check for existing content in bot-specific storage
2016 + // ================================
2017 + // FIXED: Check for existing content in ONLY the active storage method
2018 + // ================================
2019 +
4165 2020 $is_update = false;
4166 2021
4167 - // Get bot-specific Pinecone configuration
4168 - $bot_pinecone_config = $this->get_bot_pinecone_config($bot_id);
4169 - $use_pinecone = !empty($bot_pinecone_config) && ($bot_pinecone_config['use_pinecone'] ?? false);
2022 + // Check if Pinecone is enabled
2023 + $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
2024 + $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
4170 2025
4171 - if ($use_pinecone && !empty($bot_pinecone_config['api_key'])) {
4172 - // Check Pinecone for this bot
4173 - $pinecone_data = $this->mxchat_get_pinecone_processed_content($bot_pinecone_config);
2026 + if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
2027 + // ONLY check Pinecone if it's enabled
2028 + $pinecone_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
4174 2029 if (isset($pinecone_data[$post_id])) {
4175 2030 $is_update = true;
4176 2031 }
4177 2032 } else {
4178 - // Check WordPress DB (same as before since it's shared)
2033 + // ONLY check WordPress DB if Pinecone is not enabled
4179 2034 global $wpdb;
4180 2035 $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
4181 2036 $existing_record = $wpdb->get_row($wpdb->prepare(
4182 2037 "SELECT id FROM $table_name WHERE source_url = %s",
@@ -4186,54 +2041,54 @@
4186 2041 if ($existing_record) {
4187 2042 $is_update = true;
4188 2043 }
4189 2044 }
4190 -
4191 - // UPDATED 2.5.6: Determine content type based on post_type
4192 - $post_type = $post->post_type;
4193 - $content_type = 'content'; // Default fallback
4194 -
4195 - // Map WordPress post types to content types
4196 - switch ($post_type) {
4197 - case 'post':
4198 - $content_type = 'post';
4199 - break;
4200 - case 'page':
4201 - $content_type = 'page';
4202 - break;
4203 - case 'product':
4204 - $content_type = 'product';
4205 - break;
4206 - default:
4207 - // For custom post types, use the post type name
4208 - $content_type = sanitize_key($post_type);
4209 - break;
4210 - }
4211 -
4212 - // Use the centralized utility function with bot_id and content_type
2045 +
2046 + // Use the centralized utility function for storage
4213 2047 $result = MxChat_Utils::submit_content_to_db(
4214 - $content,
4215 - $source_url,
2048 + $content,
2049 + $source_url,
4216 2050 $api_key,
4217 - $vector_id,
4218 - $bot_id,
4219 - $content_type
2051 + $vector_id
4220 2052 );
4221 -
2053 +
4222 2054 if (is_wp_error($result)) {
4223 - MxChat_Admin::mxchat_log_debug('storage_error', 'Knowledge storage failed: ' . $result->get_error_message(), array('source_url' => $source_url));
4224 2055 wp_send_json_error('Storage failed: ' . $result->get_error_message());
4225 2056 exit;
4226 2057 }
4227 -
4228 - // Automatically apply role restriction based on tags
4229 - $this->apply_role_restriction_to_post($post_id, $source_url);
4230 -
2058 +
2059 + // ================================
2060 + // UPDATE: Only update caches if Pinecone is enabled
2061 + // ================================
2062 +
2063 + if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
2064 + // Update vector ID cache for improved fetching
2065 + $this->mxchat_update_pinecone_vector_cache($vector_id);
2066 +
2067 + // Update local processed content cache for immediate UI feedback
2068 + $pinecone_cache = get_option('mxchat_pinecone_processed_cache', array());
2069 + $pinecone_cache[$post_id] = array(
2070 + 'db_id' => $vector_id,
2071 + 'processed_date' => 'Just now',
2072 + 'url' => $source_url,
2073 + 'source' => 'pinecone',
2074 + 'timestamp' => current_time('timestamp')
2075 + );
2076 + update_option('mxchat_pinecone_processed_cache', $pinecone_cache);
2077 +
2078 + // Also update the general processed content cache
2079 + $processed_cache = get_option('mxchat_processed_content_cache', array());
2080 + $processed_cache[$post_id] = array(
2081 + 'db_id' => $vector_id,
2082 + 'timestamp' => current_time('timestamp'),
2083 + 'url' => $source_url,
2084 + 'source' => 'pinecone'
2085 + );
2086 + update_option('mxchat_processed_content_cache', $processed_cache);
2087 + }
2088 +
4231 2089 $operation_type = $is_update ? 'update' : 'new';
4232 2090
4233 - // Count ACF fields for debugging
4234 - $acf_field_count = count($acf_fields);
4235 -
4236 2091 // Success response with minimal data
4237 2092 wp_send_json_success(array(
4238 2093 'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
4239 2094 'post_id' => $post_id,
@@ -4238,95 +2093,27 @@
4238 2093 'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
4239 2094 'post_id' => $post_id,
4240 2095 'title' => $post->post_title,
4241 2096 'operation_type' => $operation_type,
4242 - 'vector_id' => $vector_id,
4243 - 'acf_fields_found' => $acf_field_count,
4244 - 'pdf_extracted_count' => (int) $pdf_extracted_count,
4245 - 'content_preview' => substr($content, 0, 100) . '...',
4246 - 'bot_id' => $bot_id
2097 + 'vector_id' => $vector_id, // Include vector ID for debugging
2098 + 'cache_updated' => $use_pinecone // Indicate if cache was updated
4247 2099 ));
4248 2100 exit;
4249 2101 }
4250 2102
4251 -private function apply_role_restriction_to_post($post_id, $source_url) {
4252 - // Get tag-role mappings
4253 - $mappings = get_option('mxchat_tag_role_mappings', array());
4254 -
4255 - if (empty($mappings)) {
4256 - return; // No mappings, leave as public
4257 - }
4258 -
4259 - // Get all tags for the post
4260 - $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
4261 -
4262 - if (empty($post_tags)) {
4263 - return; // No tags, leave as public
4264 - }
4265 -
4266 - // Determine the highest role restriction based on tags
4267 - $highest_role = 'public';
4268 - $role_hierarchy = array(
4269 - 'public' => 0,
4270 - 'logged_in' => 1,
4271 - 'subscriber' => 2,
4272 - 'contributor' => 3,
4273 - 'author' => 4,
4274 - 'editor' => 5,
4275 - 'administrator' => 6
4276 - );
4277 -
4278 - foreach ($post_tags as $tag_slug) {
4279 - if (isset($mappings[$tag_slug])) {
4280 - $role = $mappings[$tag_slug];
4281 - if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
4282 - $highest_role = $role;
4283 - }
4284 - }
4285 - }
4286 -
4287 - // If no restricted tags found, return (leave as public)
4288 - if ($highest_role === 'public') {
4289 - return;
4290 - }
4291 -
4292 - // Update the role restriction in the database
4293 - global $wpdb;
4294 -
4295 - // Check if using Pinecone
4296 - $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
4297 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
4298 -
4299 - if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
4300 - // Update Pinecone role restriction
4301 - $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
4302 - $vector_id = md5($source_url);
4303 -
4304 - $wpdb->replace(
4305 - $roles_table,
4306 - array(
4307 - 'vector_id' => $vector_id,
4308 - 'role_restriction' => $highest_role,
4309 - 'updated_at' => current_time('mysql')
4310 - ),
4311 - array('%s', '%s', '%s')
4312 - );
4313 - } else {
4314 - // Update WordPress DB
4315 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
4316 -
4317 - $wpdb->update(
4318 - $table_name,
4319 - array('role_restriction' => $highest_role),
4320 - array('source_url' => $source_url),
4321 - array('%s'),
4322 - array('%s')
4323 - );
4324 - }
4325 -}
4326 2103
2104 +
2105 + /**
2106 + * Updates cache with new vector ID if absent
2107 + */
2108 + public function mxchat_update_pinecone_vector_cache($vector_id) {
2109 + $cached_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
2110 + if (!in_array($vector_id, $cached_ids)) {
2111 + $cached_ids[] = $vector_id;
2112 + update_option('mxchat_pinecone_vector_ids_cache', $cached_ids);
2113 + }
2114 + }
4327 2115 public function mxchat_get_public_post_types() {
4328 - // Get all public post types
4329 2116 $post_types = get_post_types(array('public' => true), 'objects');
4330 2117 $post_type_options = array();
4331 2118
4332 2119 foreach ($post_types as $post_type) {
@@ -4332,45 +2119,51 @@
4332 2119 foreach ($post_types as $post_type) {
4333 2120 $post_type_options[$post_type->name] = $post_type->label;
4334 2121 }
4335 2122
4336 - // Also include common forum/community post types that might not be marked as public
4337 - $additional_types = array(
4338 - 'topic' => 'Forum Topics (bbPress)',
4339 - 'reply' => 'Forum Replies (bbPress)',
4340 - 'forum' => 'Forums (bbPress)',
4341 - 'wpforo_topic' => 'wpForo Topics',
4342 - 'wpforo_post' => 'wpForo Posts'
4343 - );
4344 -
4345 - foreach ($additional_types as $type_name => $type_label) {
4346 - if (post_type_exists($type_name) && !isset($post_type_options[$type_name])) {
4347 - $post_type_options[$type_name] = $type_label;
4348 - }
4349 - }
4350 -
4351 2123 return $post_type_options;
4352 2124 }
2125 +public function mxchat_get_pinecone_processed_content($pinecone_options) {
2126 + //error_log('=== DEBUG: Starting mxchat_get_pinecone_processed_content ===');
2127 +
2128 + // First check local cache for immediate updates
2129 + $cached_data = get_option('mxchat_pinecone_processed_cache', array());
2130 + //error_log('DEBUG: Found ' . count($cached_data) . ' items in local cache');
4353 2131
4354 -/**
4355 - * Retrieves processed content from Pinecone API
4356 - */
4357 -public function mxchat_get_pinecone_processed_content($pinecone_options) {
4358 2132 $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4359 2133 $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
2134 +
2135 + //error_log('DEBUG: API key present: ' . (!empty($api_key) ? 'YES' : 'NO'));
2136 + //error_log('DEBUG: Host: ' . $host);
4360 2137
4361 2138 if (empty($api_key) || empty($host)) {
4362 - return array();
2139 + //error_log('DEBUG: Missing API credentials, returning cached data only');
2140 + return $cached_data;
4363 2141 }
4364 2142
4365 2143 $pinecone_data = array();
4366 2144
4367 2145 try {
4368 - // Always get fresh data from Pinecone
4369 - $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
2146 + // Method 1: Try to get vectors using cached vector IDs first
2147 + $cached_vector_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
2148 + //error_log('DEBUG: Found ' . count($cached_vector_ids) . ' cached vector IDs');
4370 2149
4371 - // Method 2: Final fallback - try stats endpoint (if available)
2150 + if (!empty($cached_vector_ids)) {
2151 + //error_log('DEBUG: Trying to fetch by cached vector IDs...');
2152 + $pinecone_data = $this->mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $cached_vector_ids);
2153 + //error_log('DEBUG: Fetch by IDs returned ' . count($pinecone_data) . ' items');
2154 + }
2155 +
2156 + // Method 2: If no cached IDs or fetch failed, use scanning approach
4372 2157 if (empty($pinecone_data)) {
2158 + //error_log('DEBUG: Trying scanning approach...');
2159 + $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
2160 + //error_log('DEBUG: Scanning returned ' . count($pinecone_data) . ' items');
2161 + }
2162 +
2163 + // Method 3: Final fallback - try stats endpoint
2164 + if (empty($pinecone_data)) {
2165 + //error_log('DEBUG: Trying stats endpoint...');
4373 2166 $stats_url = "https://{$host}/describe_index_stats";
4374 2167
4375 2168 $response = wp_remote_post($stats_url, array(
4376 2169 'headers' => array(
@@ -4383,17 +2176,44 @@
4383 2176
4384 2177 if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
4385 2178 $body = wp_remote_retrieve_body($response);
4386 2179 $stats_data = json_decode($body, true);
2180 + //error_log('DEBUG: Pinecone stats: ' . print_r($stats_data, true));
2181 + } else {
2182 + if (is_wp_error($response)) {
2183 + //error_log('DEBUG: Stats endpoint error: ' . $response->get_error_message());
2184 + } else {
2185 + //error_log('DEBUG: Stats endpoint failed with code: ' . wp_remote_retrieve_response_code($response));
2186 + }
4387 2187 }
4388 2188 }
4389 2189
4390 2190 } catch (Exception $e) {
4391 - // Log error but return fresh data only
2191 + //error_log('DEBUG: Exception in get_pinecone_processed_content: ' . $e->getMessage());
4392 2192 }
4393 2193
4394 - return $pinecone_data;
2194 + // Merge cached data with Pinecone data
2195 + $merged_data = $pinecone_data;
2196 +
2197 + foreach ($cached_data as $post_id => $cache_item) {
2198 + $cache_timestamp = $cache_item['timestamp'] ?? 0;
2199 + $time_diff = current_time('timestamp') - $cache_timestamp;
2200 +
2201 + if ($time_diff < 300) { // 5 minutes = 300 seconds
2202 + $merged_data[$post_id] = $cache_item;
2203 + } else {
2204 + if (!isset($merged_data[$post_id])) {
2205 + $merged_data[$post_id] = $cache_item;
2206 + }
2207 + }
2208 + }
2209 +
2210 + //error_log('DEBUG: Final merged data count: ' . count($merged_data));
2211 + //error_log('=== DEBUG: End mxchat_get_pinecone_processed_content ===');
2212 +
2213 + return $merged_data;
4395 2214 }
2215 +
4396 2216 public function mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $vector_ids) {
4397 2217 //error_log('=== DEBUG: Starting mxchat_fetch_pinecone_vectors_by_ids ===');
4398 2218
4399 2219 $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
@@ -4485,46 +2305,16 @@
4485 2305 return array();
4486 2306 }
4487 2307 }
4488 2308
4489 -/**
4490 - * Get embedding dimensions based on the selected model.
4491 - */
4492 -private function mxchat_get_embedding_dimensions() {
4493 - $options = get_option('mxchat_options', array());
4494 - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4495 -
4496 - $model_dimensions = array(
4497 - 'text-embedding-ada-002' => 1536,
4498 - 'text-embedding-3-small' => 1536,
4499 - 'text-embedding-3-large' => 3072,
4500 - 'voyage-2' => 1024,
4501 - 'voyage-large-2' => 1536,
4502 - 'voyage-3-large' => 2048,
4503 - 'gemini-embedding-001' => 1536,
4504 - );
4505 -
4506 - if (strpos($selected_model, 'voyage-3-large') === 0) {
4507 - $custom_dimensions = $options['voyage_output_dimension'] ?? 2048;
4508 - return intval($custom_dimensions);
4509 - }
4510 -
4511 - if (strpos($selected_model, 'gemini-embedding') === 0) {
4512 - $custom_dimensions = $options['gemini_output_dimension'] ?? 1536;
4513 - return intval($custom_dimensions);
4514 - }
4515 -
4516 - return $model_dimensions[$selected_model] ?? 1536;
4517 -}
4518 -
4519 -/**
4520 - * Scan Pinecone for processed content
4521 - */
4522 2309 public function mxchat_scan_pinecone_for_processed_content($pinecone_options) {
2310 + //error_log('=== DEBUG: Starting mxchat_scan_pinecone_for_processed_content ===');
2311 +
4523 2312 $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4524 2313 $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
4525 2314
4526 2315 if (empty($api_key) || empty($host)) {
2316 + //error_log('DEBUG: Missing API credentials for scanning');
4527 2317 return array();
4528 2318 }
4529 2319
4530 2320 try {
@@ -4531,18 +2321,17 @@
4531 2321 // Use multiple random vectors to get better coverage
4532 2322 $all_matches = array();
4533 2323 $seen_ids = array();
4534 2324
4535 - // Get correct dimensions for the configured embedding model
4536 - $dimensions = $this->mxchat_get_embedding_dimensions();
4537 -
4538 2325 // Try 3 different random vectors to get better coverage
4539 2326 for ($i = 0; $i < 3; $i++) {
2327 + //error_log('DEBUG: Scanning attempt ' . ($i + 1) . '/3');
2328 +
4540 2329 $query_url = "https://{$host}/query";
4541 2330
4542 2331 // Generate a random unit vector instead of zeros
4543 2332 $random_vector = array();
4544 - for ($j = 0; $j < $dimensions; $j++) {
2333 + for ($j = 0; $j < 1536; $j++) {
4545 2334 $random_vector[] = (rand(-1000, 1000) / 1000.0);
4546 2335 }
4547 2336
4548 2337 // Normalize the vector to unit length
@@ -4567,14 +2356,18 @@
4567 2356 'timeout' => 30
4568 2357 ));
4569 2358
4570 2359 if (is_wp_error($response)) {
2360 + //error_log('DEBUG: Query attempt ' . ($i + 1) . ' WP error: ' . $response->get_error_message());
4571 2361 continue;
4572 2362 }
4573 2363
4574 2364 $response_code = wp_remote_retrieve_response_code($response);
2365 + //error_log('DEBUG: Query attempt ' . ($i + 1) . ' response code: ' . $response_code);
4575 2366
4576 2367 if ($response_code !== 200) {
2368 + $error_body = wp_remote_retrieve_body($response);
2369 + //error_log('DEBUG: Query attempt ' . ($i + 1) . ' failed with body: ' . substr($error_body, 0, 500));
4577 2370 continue;
4578 2371 }
4579 2372
4580 2373 $body = wp_remote_retrieve_body($response);
@@ -4580,8 +2373,9 @@
4580 2373 $body = wp_remote_retrieve_body($response);
4581 2374 $data = json_decode($body, true);
4582 2375
4583 2376 if (isset($data['matches'])) {
2377 + //error_log('DEBUG: Query attempt ' . ($i + 1) . ' returned ' . count($data['matches']) . ' matches');
4584 2378 foreach ($data['matches'] as $match) {
4585 2379 $match_id = $match['id'] ?? '';
4586 2380 if (!empty($match_id) && !isset($seen_ids[$match_id])) {
4587 2381 $all_matches[] = $match;
@@ -4587,14 +2381,18 @@
4587 2381 $all_matches[] = $match;
4588 2382 $seen_ids[$match_id] = true;
4589 2383 }
4590 2384 }
2385 + } else {
2386 + //error_log('DEBUG: Query attempt ' . ($i + 1) . ' - no matches key in response');
4591 2387 }
4592 2388 }
4593 2389
4594 - // Convert matches to processed data format, grouping by URL to count chunks
2390 + //error_log('DEBUG: Total unique matches found: ' . count($all_matches));
2391 +
2392 + // Convert matches to processed data format
4595 2393 $processed_data = array();
4596 - $url_chunk_counts = array();
2394 + $vector_ids_for_cache = array();
4597 2395
4598 2396 foreach ($all_matches as $match) {
4599 2397 $metadata = $match['metadata'] ?? array();
4600 2398 $source_url = $metadata['source_url'] ?? '';
@@ -4602,14 +2400,8 @@
4602 2400
4603 2401 if (!empty($source_url) && !empty($match_id)) {
4604 2402 $post_id = url_to_postid($source_url);
4605 2403 if ($post_id) {
4606 - // Count chunks per post_id
4607 - if (!isset($url_chunk_counts[$post_id])) {
4608 - $url_chunk_counts[$post_id] = 0;
4609 - }
4610 - $url_chunk_counts[$post_id]++;
4611 -
4612 2404 $created_at = $metadata['created_at'] ?? '';
4613 2405 $processed_date = 'Recently';
4614 2406
4615 2407 if (!empty($created_at)) {
@@ -4618,322 +2410,243 @@
4618 2410 $processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
4619 2411 }
4620 2412 }
4621 2413
4622 - // Only store if not already set, or update with newer timestamp
4623 - if (!isset($processed_data[$post_id]) ||
4624 - ($timestamp ?? 0) > ($processed_data[$post_id]['timestamp'] ?? 0)) {
4625 - $processed_data[$post_id] = array(
4626 - 'db_id' => $match_id,
4627 - 'processed_date' => $processed_date,
4628 - 'url' => $source_url,
4629 - 'source' => 'pinecone',
4630 - 'timestamp' => $timestamp ?? current_time('timestamp')
4631 - );
4632 - }
2414 + $processed_data[$post_id] = array(
2415 + 'db_id' => $match_id,
2416 + 'processed_date' => $processed_date,
2417 + 'url' => $source_url,
2418 + 'source' => 'pinecone',
2419 + 'timestamp' => $timestamp ?? current_time('timestamp')
2420 + );
2421 +
2422 + $vector_ids_for_cache[] = $match_id;
4633 2423 }
4634 2424 }
4635 2425 }
4636 2426
4637 - // Add chunk counts to processed data
4638 - foreach ($url_chunk_counts as $post_id => $chunk_count) {
4639 - if (isset($processed_data[$post_id])) {
4640 - $processed_data[$post_id]['chunk_count'] = $chunk_count;
4641 - }
2427 + // Update the vector IDs cache for future use
2428 + if (!empty($vector_ids_for_cache)) {
2429 + update_option('mxchat_pinecone_vector_ids_cache', $vector_ids_for_cache);
2430 + //error_log('DEBUG: Updated vector IDs cache with ' . count($vector_ids_for_cache) . ' IDs');
4642 2431 }
4643 2432
2433 + //error_log('DEBUG: Returning ' . count($processed_data) . ' processed items from scanning');
4644 2434 return $processed_data;
4645 2435
4646 2436 } catch (Exception $e) {
2437 + //error_log('DEBUG: Exception in scan_pinecone_for_processed_content: ' . $e->getMessage());
4647 2438 return array();
4648 2439 }
4649 2440 }
4650 -/**
4651 - * Generate embeddings from input text for MXChat with bot support
4652 - */
4653 -private function mxchat_generate_embedding($text, $bot_id = 'default') {
4654 - // Enable detailed logging for debugging
4655 - //error_log('[MXCHAT-EMBED] Starting embedding generation for bot: ' . $bot_id . '. Text length: ' . strlen($text) . ' bytes');
4656 - //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4657 2441
4658 - // Get bot-specific options
4659 - $bot_options = $this->get_bot_options($bot_id);
4660 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
2442 + /**
2443 + * Generates embeddings from input text for MXChat
2444 + */
2445 + private function mxchat_generate_embedding($text) {
2446 + // Enable detailed logging for debugging
2447 + //error_log('[MXCHAT-EMBED] Starting embedding generation. Text length: ' . strlen($text) . ' bytes');
2448 + //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4661 2449
4662 - // Opt-in: when the custom provider is selected for embeddings, index through
4663 - // the same custom endpoint the query path uses so stored vectors and query
4664 - // vectors share a model. Returns the vector array on success, or an error
4665 - // string on failure (this function's existing failure contract).
4666 - if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
4667 - if (!class_exists('MxChat_Utils')) {
4668 - require_once dirname(__FILE__) . '/../includes/class-mxchat-utils.php';
4669 - }
4670 - return MxChat_Utils::generate_embedding_custom($text, $options);
4671 - }
2450 + $options = get_option('mxchat_options');
2451 + $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
2452 + //error_log('[MXCHAT-EMBED] Selected embedding model: ' . $selected_model);
4672 2453
4673 - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4674 - //error_log('[MXCHAT-EMBED] Selected embedding model for bot ' . $bot_id . ': ' . $selected_model);
2454 + // Determine provider and endpoint
2455 + if (strpos($selected_model, 'voyage') === 0) {
2456 + $api_key = $options['voyage_api_key'] ?? '';
2457 + $endpoint = 'https://api.voyageai.com/v1/embeddings';
2458 + $provider_name = 'Voyage AI';
2459 + //error_log('[MXCHAT-EMBED] Using Voyage AI API');
2460 + } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
2461 + $api_key = $options['gemini_api_key'] ?? '';
2462 + $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
2463 + $provider_name = 'Google Gemini';
2464 + //error_log('[MXCHAT-EMBED] Using Google Gemini API');
2465 + } else {
2466 + $api_key = $options['api_key'] ?? '';
2467 + $endpoint = 'https://api.openai.com/v1/embeddings';
2468 + $provider_name = 'OpenAI';
2469 + //error_log('[MXCHAT-EMBED] Using OpenAI API');
2470 + }
4675 2471
4676 - // Determine provider and endpoint
4677 - if (strpos($selected_model, 'voyage') === 0) {
4678 - $api_key = $options['voyage_api_key'] ?? '';
4679 - $endpoint = 'https://api.voyageai.com/v1/embeddings';
4680 - $provider_name = 'Voyage AI';
4681 - //error_log('[MXCHAT-EMBED] Using Voyage AI API for bot ' . $bot_id);
4682 - } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
4683 - $api_key = $options['gemini_api_key'] ?? '';
4684 - $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
4685 - $provider_name = 'Google Gemini';
4686 - //error_log('[MXCHAT-EMBED] Using Google Gemini API for bot ' . $bot_id);
4687 - } else {
4688 - $api_key = $options['api_key'] ?? '';
4689 - $endpoint = 'https://api.openai.com/v1/embeddings';
4690 - $provider_name = 'OpenAI';
4691 - //error_log('[MXCHAT-EMBED] Using OpenAI API for bot ' . $bot_id);
4692 - }
2472 + //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
4693 2473
4694 - //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
2474 + if (empty($api_key)) {
2475 + $error_message = sprintf('Missing %s API key. Please configure your API key in the MxChat settings.', $provider_name);
2476 + //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
2477 + return $error_message;
2478 + }
4695 2479
4696 - if (empty($api_key)) {
4697 - $error_message = sprintf('Missing %s API key for bot %s. Please configure your API key in the bot settings.', $provider_name, $bot_id);
4698 - //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
4699 - return $error_message;
4700 - }
2480 + // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
2481 + $estimated_tokens = ceil(str_word_count($text) / 0.75);
2482 + //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
4701 2483
4702 - // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
4703 - $estimated_tokens = ceil(str_word_count($text) / 0.75);
4704 - //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
2484 + if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
2485 + //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
2486 + // Consider truncating text here
2487 + }
4705 2488
4706 - if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
4707 - //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
4708 - // Consider truncating text here
4709 - }
2489 + // Prepare request body based on provider
2490 + if (strpos($selected_model, 'gemini-embedding') === 0) {
2491 + // Gemini API format
2492 + $request_body = array(
2493 + 'model' => 'models/' . $selected_model,
2494 + 'content' => array(
2495 + 'parts' => array(
2496 + array('text' => $text)
2497 + )
2498 + )
2499 + );
4710 2500
4711 - // Prepare request body based on provider
4712 - if (strpos($selected_model, 'gemini-embedding') === 0) {
4713 - // Gemini API format
4714 - $request_body = array(
4715 - 'model' => 'models/' . $selected_model,
4716 - 'content' => array(
4717 - 'parts' => array(
4718 - array('text' => $text)
4719 - )
4720 - )
4721 - );
2501 + // Set output dimensionality to 1536 for consistency with other models
2502 + $request_body['outputDimensionality'] = 1536;
2503 + } else {
2504 + // OpenAI/Voyage API format
2505 + $request_body = array(
2506 + 'model' => $selected_model,
2507 + 'input' => $text
2508 + );
4722 2509
4723 - // Set output dimensionality to 1536 for consistency with other models
4724 - $request_body['outputDimensionality'] = 1536;
4725 - } else {
4726 - // OpenAI/Voyage API format
4727 - $request_body = array(
4728 - 'model' => $selected_model,
4729 - 'input' => $text
4730 - );
2510 + // Add output_dimension for voyage-3-large model
2511 + if ($selected_model === 'voyage-3-large') {
2512 + $request_body['output_dimension'] = 2048;
2513 + }
2514 + }
4731 2515
4732 - // Add output_dimension for voyage-3-large model
4733 - if ($selected_model === 'voyage-3-large') {
4734 - $request_body['output_dimension'] = 2048;
4735 - }
4736 - }
2516 + //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
4737 2517
4738 - //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
2518 + // Prepare headers based on provider
2519 + if (strpos($selected_model, 'gemini-embedding') === 0) {
2520 + // Gemini uses API key as query parameter
2521 + $endpoint .= '?key=' . $api_key;
2522 + $headers = array(
2523 + 'Content-Type' => 'application/json'
2524 + );
2525 + } else {
2526 + // OpenAI/Voyage use Bearer token
2527 + $headers = array(
2528 + 'Authorization' => 'Bearer ' . $api_key,
2529 + 'Content-Type' => 'application/json'
2530 + );
2531 + }
4739 2532
4740 - // Prepare headers based on provider
4741 - if (strpos($selected_model, 'gemini-embedding') === 0) {
4742 - // Gemini uses API key as query parameter
4743 - $endpoint .= '?key=' . $api_key;
4744 - $headers = array(
4745 - 'Content-Type' => 'application/json'
4746 - );
4747 - } else {
4748 - // OpenAI/Voyage use Bearer token
4749 - $headers = array(
4750 - 'Authorization' => 'Bearer ' . $api_key,
4751 - 'Content-Type' => 'application/json'
4752 - );
4753 - }
2533 + // Make API request
2534 + //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
2535 + $response = wp_remote_post($endpoint, array(
2536 + 'body' => wp_json_encode($request_body),
2537 + 'headers' => $headers,
2538 + 'timeout' => 60 // Increased timeout for large inputs
2539 + ));
4754 2540
4755 - // Make API request
4756 - //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
4757 - $response = wp_remote_post($endpoint, array(
4758 - 'body' => wp_json_encode($request_body),
4759 - 'headers' => $headers,
4760 - 'timeout' => 60 // Increased timeout for large inputs
4761 - ));
2541 + // Handle wp_remote_post errors
2542 + if (is_wp_error($response)) {
2543 + $error_message = $response->get_error_message();
2544 + //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
2545 + return 'Connection error: ' . $error_message;
2546 + }
4762 2547
4763 - // Handle wp_remote_post errors
4764 - if (is_wp_error($response)) {
4765 - $error_message = $response->get_error_message();
4766 - //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
4767 - return 'Connection error: ' . $error_message;
4768 - }
2548 + // Get and check HTTP response code
2549 + $http_code = wp_remote_retrieve_response_code($response);
2550 + //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
4769 2551
4770 - // Get and check HTTP response code
4771 - $http_code = wp_remote_retrieve_response_code($response);
4772 - //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
2552 + if ($http_code !== 200) {
2553 + $error_body = wp_remote_retrieve_body($response);
2554 + //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
4773 2555
4774 - if ($http_code !== 200) {
4775 - $error_body = wp_remote_retrieve_body($response);
4776 - //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
2556 + // Try to parse error for more details
2557 + $error_json = json_decode($error_body, true);
2558 + if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
2559 + $error_type = $error_json['error']['type'] ?? 'unknown';
2560 + $error_message = $error_json['error']['message'] ?? 'No message';
2561 + //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
2562 + //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
4777 2563
4778 - // Try to parse error for more details
4779 - $error_json = json_decode($error_body, true);
4780 - if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
4781 - $error_type = $error_json['error']['type'] ?? 'unknown';
4782 - $error_message = $error_json['error']['message'] ?? 'No message';
4783 - //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
4784 - //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
2564 + // Customize error message for common API errors
2565 + if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
2566 + $error_message = sprintf('Invalid %s API key. Please check your API key in the MxChat settings.', $provider_name);
2567 + } elseif ($error_type === 'authentication_error') {
2568 + $error_message = sprintf('%s authentication failed. Please verify your API key in the MxChat settings.', $provider_name);
2569 + }
4785 2570
4786 - // Customize error message for common API errors
4787 - if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
4788 - $error_message = sprintf('Invalid %s API key for bot %s. Please check your API key in the bot settings.', $provider_name, $bot_id);
4789 - } elseif ($error_type === 'authentication_error') {
4790 - $error_message = sprintf('%s authentication failed for bot %s. Please verify your API key in the bot settings.', $provider_name, $bot_id);
4791 - }
2571 + //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
2572 + return $error_message;
2573 + }
4792 2574
4793 - //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4794 - return $error_message;
4795 - }
2575 + $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding", $http_code);
2576 + //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
2577 + return $error_message;
2578 + }
4796 2579
4797 - $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding for bot %s", $http_code, $bot_id);
4798 - //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4799 - return $error_message;
4800 - }
2580 + // Parse response body
2581 + $response_body = wp_remote_retrieve_body($response);
2582 + //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
4801 2583
4802 - // Parse response body
4803 - $response_body = wp_remote_retrieve_body($response);
4804 - //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
2584 + $response_data = json_decode($response_body, true);
4805 2585
4806 - $response_data = json_decode($response_body, true);
2586 + if (json_last_error() !== JSON_ERROR_NONE) {
2587 + $error = json_last_error_msg();
2588 + //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
2589 + //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
2590 + return "Failed to parse API response: $error";
2591 + }
4807 2592
4808 - if (json_last_error() !== JSON_ERROR_NONE) {
4809 - $error = json_last_error_msg();
4810 - //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
4811 - //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
4812 - return "Failed to parse API response: $error";
4813 - }
2593 + // Handle different response formats based on provider
2594 + if (strpos($selected_model, 'gemini-embedding') === 0) {
2595 + // Gemini API response format
2596 + if (isset($response_data['embedding']['values'])) {
2597 + $embedding_dimensions = count($response_data['embedding']['values']);
2598 + //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
4814 2599
4815 - // Handle different response formats based on provider
4816 - if (strpos($selected_model, 'gemini-embedding') === 0) {
4817 - // Gemini API response format
4818 - if (isset($response_data['embedding']['values'])) {
4819 - $embedding_dimensions = count($response_data['embedding']['values']);
4820 - //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
2600 + // Check if embedding dimensions are as expected (should be 1536)
2601 + if ($embedding_dimensions !== 1536) {
2602 + //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
2603 + }
4821 2604
4822 - // Check if embedding dimensions are as expected (should be 1536)
4823 - if ($embedding_dimensions !== 1536) {
4824 - //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
4825 - }
2605 + return $response_data['embedding']['values'];
2606 + } else {
2607 + //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
2608 + //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4826 2609
4827 - return $response_data['embedding']['values'];
4828 - } else {
4829 - //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
4830 - //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
2610 + if (isset($response_data['error'])) {
2611 + $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
2612 + //error_log('[MXCHAT-EMBED] ' . $error_message);
2613 + return $error_message;
2614 + }
4831 2615
4832 - if (isset($response_data['error'])) {
4833 - $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
4834 - //error_log('[MXCHAT-EMBED] ' . $error_message);
4835 - return $error_message;
4836 - }
2616 + $error_message = "Invalid Gemini API response format: No embedding found";
2617 + //error_log('[MXCHAT-EMBED] ' . $error_message);
2618 + return $error_message;
2619 + }
2620 + } else {
2621 + // OpenAI/Voyage API response format
2622 + if (isset($response_data['data'][0]['embedding'])) {
2623 + $embedding_dimensions = count($response_data['data'][0]['embedding']);
2624 + //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
4837 2625
4838 - $error_message = "Invalid Gemini API response format: No embedding found";
4839 - //error_log('[MXCHAT-EMBED] ' . $error_message);
4840 - return $error_message;
4841 - }
4842 - } else {
4843 - // OpenAI/Voyage API response format
4844 - if (isset($response_data['data'][0]['embedding'])) {
4845 - $embedding_dimensions = count($response_data['data'][0]['embedding']);
4846 - //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
2626 + // Check if embedding dimensions are as expected
2627 + if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
2628 + ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
2629 + //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
2630 + }
4847 2631
4848 - // Check if embedding dimensions are as expected
4849 - if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
4850 - ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
4851 - //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
4852 - }
2632 + return $response_data['data'][0]['embedding'];
2633 + } else {
2634 + //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
2635 + //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4853 2636
4854 - return $response_data['data'][0]['embedding'];
4855 - } else {
4856 - //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
4857 - //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
2637 + if (isset($response_data['error'])) {
2638 + $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
2639 + //error_log('[MXCHAT-EMBED] ' . $error_message);
2640 + return $error_message;
2641 + }
4858 2642
4859 - if (isset($response_data['error'])) {
4860 - $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
4861 - //error_log('[MXCHAT-EMBED] ' . $error_message);
4862 - return $error_message;
4863 - }
4864 -
4865 - $error_message = "Invalid API response format: No embedding found";
4866 - //error_log('[MXCHAT-EMBED] ' . $error_message);
4867 - return $error_message;
4868 - }
4869 - }
4870 -}
4871 -
4872 -/**
4873 - * Get bot-specific options for multi-bot functionality
4874 - * Falls back to default options if bot_id is 'default' or multi-bot add-on is not active
4875 - */
4876 -private function get_bot_options($bot_id = 'default') {
4877 - //error_log("MXCHAT DEBUG: get_bot_options called for bot: " . $bot_id);
4878 -
4879 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4880 - //error_log("MXCHAT DEBUG: Using default options (no multi-bot or bot is 'default')");
4881 - return array();
4882 - }
4883 -
4884 - $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
4885 -
4886 - if (!empty($bot_options)) {
4887 - //error_log("MXCHAT DEBUG: Got bot-specific options from filter");
4888 - if (isset($bot_options['similarity_threshold'])) {
4889 - //error_log(" - similarity_threshold: " . $bot_options['similarity_threshold']);
4890 - }
4891 - }
4892 -
4893 - return is_array($bot_options) ? $bot_options : array();
4894 -}
4895 -
4896 -/**
4897 - * Get bot-specific Pinecone configuration
4898 - * Used in the knowledge retrieval functions
4899 - */
4900 -// Also add debugging to your get_bot_pinecone_config function
4901 -private function get_bot_pinecone_config($bot_id = 'default') {
4902 - //error_log("MXCHAT DEBUG: get_bot_pinecone_config called for bot: " . $bot_id);
4903 -
4904 - // If default bot or multi-bot add-on not active, use default Pinecone config
4905 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4906 - //error_log("MXCHAT DEBUG: Using default Pinecone config (no multi-bot or bot is 'default')");
4907 - $addon_options = get_option('mxchat_pinecone_addon_options', array());
4908 - $config = array(
4909 - 'use_pinecone' => (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1'),
4910 - 'api_key' => $addon_options['mxchat_pinecone_api_key'] ?? '',
4911 - 'host' => $addon_options['mxchat_pinecone_host'] ?? '',
4912 - 'namespace' => $addon_options['mxchat_pinecone_namespace'] ?? ''
4913 - );
4914 - //error_log("MXCHAT DEBUG: Default config - use_pinecone: " . ($config['use_pinecone'] ? 'true' : 'false'));
4915 - return $config;
4916 - }
4917 -
4918 - //error_log("MXCHAT DEBUG: Calling filter 'mxchat_get_bot_pinecone_config' for bot: " . $bot_id);
4919 -
4920 - // Hook for multi-bot add-on to provide bot-specific Pinecone config
4921 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
4922 -
4923 - if (!empty($bot_pinecone_config)) {
4924 - //error_log("MXCHAT DEBUG: Got bot-specific config from filter");
4925 - //error_log(" - use_pinecone: " . (isset($bot_pinecone_config['use_pinecone']) ? ($bot_pinecone_config['use_pinecone'] ? 'true' : 'false') : 'not set'));
4926 - //error_log(" - host: " . ($bot_pinecone_config['host'] ?? 'not set'));
4927 - //error_log(" - namespace: " . ($bot_pinecone_config['namespace'] ?? 'not set'));
4928 - } else {
4929 - //error_log("MXCHAT DEBUG: Filter returned empty config!");
4930 - }
4931 -
4932 - return is_array($bot_pinecone_config) ? $bot_pinecone_config : array();
4933 -}
4934 -
4935 -
2643 + $error_message = "Invalid API response format: No embedding found";
2644 + //error_log('[MXCHAT-EMBED] ' . $error_message);
2645 + return $error_message;
2646 + }
2647 + }
2648 + }
4936 2649 public function mxchat_ajax_dismiss_completed_status() {
4937 2650 try {
4938 2651 // Verify the request
4939 2652 check_ajax_referer('mxchat_status_nonce', 'nonce');
@@ -5296,564 +3009,18 @@
5296 3009
5297 3010 return $html;
5298 3011 }
5299 3012
5300 -/**
5301 - * Get all ACF fields for a specific post, excluding any fields the user has disabled
5302 - */
5303 -public function mxchat_get_acf_fields_for_post($post_id) {
5304 - if (!function_exists('get_fields')) {
5305 - return array();
5306 - }
5307 -
5308 - $fields = get_fields($post_id);
5309 - if (!$fields || !is_array($fields)) {
5310 - return array();
5311 - }
5312 -
5313 - // Get excluded fields from settings
5314 - $excluded_fields = get_option('mxchat_acf_excluded_fields', array());
5315 - if (!empty($excluded_fields) && is_array($excluded_fields)) {
5316 - foreach ($excluded_fields as $excluded_field) {
5317 - if (isset($fields[$excluded_field])) {
5318 - unset($fields[$excluded_field]);
5319 - }
5320 - }
5321 - }
5322 -
5323 - return $fields;
5324 -}
5325 -
5326 -/**
5327 - * Get all registered ACF field groups and their fields for the settings UI
5328 - */
5329 -public function mxchat_get_all_acf_fields() {
5330 - if (!function_exists('acf_get_field_groups') || !function_exists('acf_get_fields')) {
5331 - return array();
5332 - }
5333 -
5334 - $all_fields = array();
5335 - $field_groups = acf_get_field_groups();
5336 -
5337 - if (!empty($field_groups)) {
5338 - foreach ($field_groups as $group) {
5339 - $group_fields = acf_get_fields($group['key']);
5340 - if (!empty($group_fields)) {
5341 - $all_fields[$group['title']] = array();
5342 - foreach ($group_fields as $field) {
5343 - $all_fields[$group['title']][] = array(
5344 - 'name' => $field['name'],
5345 - 'label' => $field['label'],
5346 - 'type' => $field['type']
5347 - );
5348 - }
5349 - }
5350 - }
5351 - }
5352 -
5353 - return $all_fields;
5354 -}
5355 -
5356 -/**
5357 - * Get whitelisted custom post meta for a given post
5358 - * This allows non-ACF meta fields (like OptionTree, theme meta boxes) to be included in embeddings
5359 - */
5360 -public function mxchat_get_whitelisted_post_meta($post_id) {
5361 - $whitelist = get_option('mxchat_custom_meta_whitelist', '');
5362 -
5363 - if (empty($whitelist)) {
5364 - return array();
5365 - }
5366 -
5367 - // Parse the whitelist - one meta key per line
5368 - $meta_keys = array_filter(array_map('trim', explode("\n", $whitelist)));
5369 -
5370 - if (empty($meta_keys)) {
5371 - return array();
5372 - }
5373 -
5374 - $result = array();
5375 -
5376 - foreach ($meta_keys as $key) {
5377 - // Skip empty keys
5378 - if (empty($key)) {
5379 - continue;
5380 - }
5381 -
5382 - $value = get_post_meta($post_id, $key, true);
5383 -
5384 - // Only include non-empty string values
5385 - if (!empty($value) && is_string($value)) {
5386 - $result[$key] = $value;
5387 - } elseif (!empty($value) && is_array($value)) {
5388 - // Handle array values by joining them
5389 - $flat_value = $this->mxchat_flatten_meta_array($value);
5390 - if (!empty($flat_value)) {
5391 - $result[$key] = $flat_value;
5392 - }
5393 - }
5394 - }
5395 -
5396 - return $result;
5397 -}
5398 -
5399 -/**
5400 - * Flatten array meta values into a readable string
5401 - */
5402 -private function mxchat_flatten_meta_array($array, $depth = 0) {
5403 - if ($depth > 3) {
5404 - return ''; // Prevent infinite recursion
5405 - }
5406 -
5407 - $parts = array();
5408 -
5409 - foreach ($array as $key => $value) {
5410 - if (is_string($value) && !empty($value)) {
5411 - $parts[] = $value;
5412 - } elseif (is_array($value)) {
5413 - $nested = $this->mxchat_flatten_meta_array($value, $depth + 1);
5414 - if (!empty($nested)) {
5415 - $parts[] = $nested;
5416 - }
5417 - }
5418 - }
5419 -
5420 - return implode(', ', $parts);
5421 -}
5422 -
5423 -/**
5424 - * Format ACF field values for content extraction
5425 - */
5426 -public function mxchat_format_acf_field_value($value, $field_name = '', $post_id = 0) {
5427 - if (empty($value)) {
5428 - return '';
5429 - }
5430 -
5431 - // Handle WP_Post objects first (THIS IS THE KEY FIX)
5432 - if ($value instanceof WP_Post) {
5433 - return $value->post_title ?: '';
5434 - }
5435 -
5436 - // Handle other WP objects
5437 - if (is_object($value)) {
5438 - if (isset($value->post_title)) {
5439 - return $value->post_title;
5440 - } elseif (isset($value->display_name)) {
5441 - return $value->display_name;
5442 - } elseif (isset($value->name)) {
5443 - return $value->name;
5444 - } elseif (method_exists($value, '__toString')) {
5445 - try {
5446 - return (string) $value;
5447 - } catch (Exception $e) {
5448 - return '';
5449 - }
5450 - }
5451 - // For any other objects, return empty string
5452 - return '';
5453 - }
5454 -
5455 - // Handle different ACF field types
5456 - if (is_array($value)) {
5457 - // Check if it's an image/file field
5458 - if (isset($value['url'])) {
5459 - // Image field - return alt text, title, or caption
5460 - if (!empty($value['alt'])) {
5461 - return $value['alt'];
5462 - } elseif (!empty($value['title'])) {
5463 - return $value['title'];
5464 - } elseif (!empty($value['caption'])) {
5465 - return $value['caption'];
5466 - } else {
5467 - return ''; // Don't include just the URL
5468 - }
5469 - }
5470 -
5471 - // Check if it's a post object or relationship field
5472 - if (isset($value['post_title'])) {
5473 - return $value['post_title'];
5474 - }
5475 -
5476 - // Check if it's a user field
5477 - if (isset($value['display_name'])) {
5478 - return $value['display_name'];
5479 - }
5480 -
5481 - // Check if it's a taxonomy term
5482 - if (isset($value['name']) && isset($value['taxonomy'])) {
5483 - return $value['name'];
5484 - }
5485 -
5486 - // Check if it's a select field with label
5487 - if (isset($value['label'])) {
5488 - return $value['label'];
5489 - }
5490 -
5491 - // Check for repeater field or flexible content
5492 - if (is_numeric(key($value))) {
5493 - $sub_values = array();
5494 - foreach ($value as $sub_item) {
5495 - if (is_array($sub_item)) {
5496 - // For repeater/flexible content, extract text values
5497 - $sub_text = $this->mxchat_extract_text_from_acf_array($sub_item);
5498 - if (!empty($sub_text)) {
5499 - $sub_values[] = $sub_text;
5500 - }
5501 - } elseif ($sub_item instanceof WP_Post) {
5502 - // Handle WP_Post objects in arrays
5503 - $sub_values[] = $sub_item->post_title ?: '';
5504 - } else {
5505 - $sub_values[] = (string) $sub_item;
5506 - }
5507 - }
5508 - return implode(', ', array_filter($sub_values));
5509 - }
5510 -
5511 - // For other arrays, try to extract meaningful text
5512 - $text_values = array();
5513 - foreach ($value as $key => $val) {
5514 - if (is_string($val) && !empty(trim($val))) {
5515 - $text_values[] = trim($val);
5516 - } elseif ($val instanceof WP_Post) {
5517 - // Handle WP_Post objects in associative arrays
5518 - $text_values[] = $val->post_title ?: '';
5519 - } elseif (is_array($val) && isset($val['post_title'])) {
5520 - $text_values[] = $val['post_title'];
5521 - } elseif (is_array($val) && isset($val['name'])) {
5522 - $text_values[] = $val['name'];
5523 - }
5524 - }
5525 -
5526 - return implode(', ', array_filter($text_values));
5527 - }
5528 -
5529 - // Handle boolean values
5530 - if (is_bool($value)) {
5531 - return $value ? 'Yes' : 'No';
5532 - }
5533 -
5534 - // Handle numeric values
5535 - if (is_numeric($value)) {
5536 - return (string) $value;
5537 - }
5538 -
5539 - // Handle string values
5540 - if (is_string($value)) {
5541 - return trim($value);
5542 - }
5543 -
5544 - // For anything else that we can't handle, return empty string
5545 - // This prevents the "Object could not be converted to string" error
5546 - return '';
5547 -}
5548 -
5549 -/**
5550 - * Extract text from complex ACF array structures
5551 - */
5552 -private function mxchat_extract_text_from_acf_array($array) {
5553 - if (!is_array($array)) {
5554 - return '';
5555 - }
5556 -
5557 - $text_parts = array();
5558 -
5559 - foreach ($array as $key => $value) {
5560 - if (is_string($value) && !empty(trim($value))) {
5561 - // Skip keys that are likely to be IDs or technical values
5562 - if (!is_numeric($value) || strlen($value) > 10) {
5563 - $text_parts[] = trim($value);
5564 - }
5565 - } elseif ($value instanceof WP_Post) {
5566 - // Handle WP_Post objects
5567 - $text_parts[] = $value->post_title ?: '';
5568 - } elseif (is_array($value)) {
5569 - if (isset($value['post_title'])) {
5570 - $text_parts[] = $value['post_title'];
5571 - } elseif (isset($value['name'])) {
5572 - $text_parts[] = $value['name'];
5573 - } elseif (isset($value['label'])) {
5574 - $text_parts[] = $value['label'];
5575 - }
5576 - } elseif (is_object($value)) {
5577 - // Handle other objects safely
5578 - if (isset($value->post_title)) {
5579 - $text_parts[] = $value->post_title;
5580 - } elseif (isset($value->name)) {
5581 - $text_parts[] = $value->name;
5582 - } elseif (isset($value->display_name)) {
5583 - $text_parts[] = $value->display_name;
5584 - }
5585 - }
5586 - }
5587 -
5588 - return implode(', ', array_filter($text_parts));
5589 -}
5590 -
5591 -/**
5592 - * Walk an ACF field value tree and collect attachment IDs for any value that
5593 - * resolves to a PDF in the WordPress media library. Handles the three shapes
5594 - * ACF returns for File/Image/URL fields (array with ID+url, integer attachment ID,
5595 - * plain URL string), and recurses through repeater/group/flexible content.
5596 - *
5597 - * @param mixed $value The ACF field value (any depth)
5598 - * @param array $out Accumulator (passed by reference) for attachment IDs
5599 - * @param int $depth Recursion guard
5600 - */
5601 -private function mxchat_collect_pdf_attachment_ids_from_acf_value($value, &$out, $depth = 0) {
5602 - if ($depth > 6) {
5603 - return; // prevent runaway recursion on circular/very-deep structures
5604 - }
5605 -
5606 - if (empty($value)) {
5607 - return;
5608 - }
5609 -
5610 - // Array shapes: ACF File/Image return value=array; repeaters/groups are arrays of arrays
5611 - if (is_array($value)) {
5612 - // Direct File/Image-style array (has 'url' and usually 'ID' + 'mime_type')
5613 - $looks_like_attachment = isset($value['url']) || isset($value['ID']) || isset($value['id']);
5614 - if ($looks_like_attachment) {
5615 - $att_id = 0;
5616 - if (!empty($value['ID']) && is_numeric($value['ID'])) {
5617 - $att_id = (int) $value['ID'];
5618 - } elseif (!empty($value['id']) && is_numeric($value['id'])) {
5619 - $att_id = (int) $value['id'];
5620 - } elseif (!empty($value['url']) && is_string($value['url'])) {
5621 - $att_id = (int) attachment_url_to_postid($value['url']);
5622 - }
5623 -
5624 - $is_pdf = false;
5625 - if (!empty($value['mime_type']) && $value['mime_type'] === 'application/pdf') {
5626 - $is_pdf = true;
5627 - } elseif (!empty($value['subtype']) && strtolower((string) $value['subtype']) === 'pdf') {
5628 - $is_pdf = true;
5629 - } elseif (!empty($value['url']) && is_string($value['url']) && $this->mxchat_url_looks_like_pdf($value['url'])) {
5630 - $is_pdf = true;
5631 - } elseif ($att_id && get_post_mime_type($att_id) === 'application/pdf') {
5632 - $is_pdf = true;
5633 - }
5634 -
5635 - if ($is_pdf && $att_id && get_post_mime_type($att_id) === 'application/pdf') {
5636 - $out[] = $att_id;
5637 - }
5638 - // An array node that represents one attachment doesn't contain other
5639 - // attachments inside it — done with this branch.
5640 - return;
5641 - }
5642 -
5643 - // Recurse: repeater rows, flexible-content layouts, groups, etc.
5644 - foreach ($value as $sub) {
5645 - $this->mxchat_collect_pdf_attachment_ids_from_acf_value($sub, $out, $depth + 1);
5646 - }
5647 - return;
5648 - }
5649 -
5650 - // Plain numeric attachment ID (ACF File field set to "Return: ID")
5651 - if (is_numeric($value)) {
5652 - $att_id = (int) $value;
5653 - if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5654 - $out[] = $att_id;
5655 - }
5656 - return;
5657 - }
5658 -
5659 - // Plain string — URL pointing at a PDF (ACF File field set to "Return: URL", or a custom URL/text field)
5660 - if (is_string($value)) {
5661 - $trimmed = trim($value);
5662 - if ($trimmed !== '' && $this->mxchat_url_looks_like_pdf($trimmed)) {
5663 - $att_id = (int) attachment_url_to_postid($trimmed);
5664 - if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5665 - $out[] = $att_id;
5666 - }
5667 - }
5668 - return;
5669 - }
5670 -}
5671 -
5672 -/**
5673 - * Heuristic: does this URL/string look like a PDF reference?
5674 - * Tolerates query strings and fragments (#page=2).
5675 - */
5676 -private function mxchat_url_looks_like_pdf($url) {
5677 - if (!is_string($url) || $url === '') {
5678 - return false;
5679 - }
5680 - // Strip query + fragment before checking extension
5681 - $path = preg_replace('/[?#].*$/', '', $url);
5682 - return (bool) preg_match('/\.pdf$/i', $path);
5683 -}
5684 -
5685 -/**
5686 - * Extract text from a PDF attachment by ID using the bundled Smalot parser.
5687 - * Reads the file directly from disk via get_attached_file (no HTTP fetch).
5688 - * Result is cached on the attachment as post_meta keyed by file mtime so we
5689 - * only parse the same PDF once unless the file changes on disk.
5690 - *
5691 - * @param int $attachment_id
5692 - * @return string Extracted plain text, or '' on failure.
5693 - */
5694 -private function mxchat_extract_pdf_text_by_attachment_id($attachment_id) {
5695 - $attachment_id = (int) $attachment_id;
5696 - if ($attachment_id <= 0) {
5697 - return '';
5698 - }
5699 - if (get_post_mime_type($attachment_id) !== 'application/pdf') {
5700 - return '';
5701 - }
5702 -
5703 - $pdf_path = get_attached_file($attachment_id);
5704 - if (empty($pdf_path) || !file_exists($pdf_path) || !is_readable($pdf_path)) {
5705 - return '';
5706 - }
5707 -
5708 - // Raw-file size cap. Parsing very large PDFs can OOM the request; skip with a log entry
5709 - // and let the rest of the ACF content land in the KB. Filterable for users who need it bigger.
5710 - $default_max_bytes = 25 * 1024 * 1024;
5711 - $max_bytes = (int) apply_filters('mxchat_acf_pdf_max_bytes', $default_max_bytes, $attachment_id, $pdf_path);
5712 - if ($max_bytes > 0) {
5713 - $file_size = @filesize($pdf_path);
5714 - if ($file_size !== false && $file_size > $max_bytes) {
5715 - error_log(sprintf(
5716 - '[mxchat] ACF PDF skipped (over size cap): attachment %d "%s" %d bytes > cap %d',
5717 - $attachment_id,
5718 - basename($pdf_path),
5719 - $file_size,
5720 - $max_bytes
5721 - ));
5722 - return '';
5723 - }
5724 - }
5725 -
5726 - $mtime = @filemtime($pdf_path);
5727 - $cache_meta_key = '_mxchat_acf_pdf_text_v1';
5728 - $cached = get_post_meta($attachment_id, $cache_meta_key, true);
5729 - if (is_array($cached) && isset($cached['mtime'], $cached['text']) && (int) $cached['mtime'] === (int) $mtime) {
5730 - return (string) $cached['text'];
5731 - }
5732 -
5733 - $text = '';
5734 - try {
5735 - if (function_exists('mxchat_load_pdf_parser')) {
5736 - mxchat_load_pdf_parser();
5737 - }
5738 - if (!class_exists('\\Smalot\\PdfParser\\Parser')) {
5739 - return '';
5740 - }
5741 - $parser = new \Smalot\PdfParser\Parser();
5742 - $pdf = $parser->parseFile($pdf_path);
5743 - $pages = $pdf->getPages();
5744 - $page_texts = array();
5745 - foreach ($pages as $page) {
5746 - $page_text = '';
5747 - try {
5748 - $page_text = $page->getText();
5749 - } catch (\Exception $e) {
5750 - $page_text = '';
5751 - }
5752 - if (!empty($page_text)) {
5753 - $page_texts[] = $page_text;
5754 - }
5755 - }
5756 - $text = trim(implode("\n\n", $page_texts));
5757 - } catch (\Exception $e) {
5758 - error_log('[mxchat] ACF PDF extraction failed for attachment ' . $attachment_id . ': ' . $e->getMessage());
5759 - return '';
5760 - } catch (\Throwable $e) {
5761 - error_log('[mxchat] ACF PDF extraction error for attachment ' . $attachment_id . ': ' . $e->getMessage());
5762 - return '';
5763 - }
5764 -
5765 - // Cap per-PDF text to avoid blowing up the embedding payload on enormous PDFs.
5766 - // The chunker downstream will still split this into multiple vectors.
5767 - $max_len = (int) apply_filters('mxchat_acf_pdf_text_max_length', 50000);
5768 - if ($max_len > 0 && strlen($text) > $max_len) {
5769 - $text = substr($text, 0, $max_len);
5770 - }
5771 -
5772 - update_post_meta($attachment_id, $cache_meta_key, array(
5773 - 'mtime' => (int) $mtime,
5774 - 'text' => $text,
5775 - ));
5776 -
5777 - return $text;
5778 -}
5779 -
5780 -/**
5781 - * Handle ACF save - fires after ACF fields are saved
5782 - * This ensures ACF field data is available when syncing to knowledge base
5783 - */
5784 -public function mxchat_handle_acf_save($post_id) {
5785 - // Skip if not a valid post
5786 - if (!$post_id || $post_id === 'options') {
5787 - return;
5788 - }
5789 -
5790 - // Skip autosaves and revisions
3013 +public function mxchat_handle_post_update($post_id, $post, $update) {
3014 + // Basic validation checks
5791 3015 if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5792 3016 return;
5793 3017 }
5794 -
5795 - $post = get_post($post_id);
5796 - if (!$post) {
5797 - return;
5798 - }
5799 -
5800 - $post_type = $post->post_type;
5801 -
5802 - // Check if sync is enabled for this post type
5803 - $should_sync = false;
5804 -
5805 - if ($post_type === 'post' && get_option('mxchat_auto_sync_posts') === '1') {
5806 - $should_sync = true;
5807 - } else if ($post_type === 'page' && get_option('mxchat_auto_sync_pages') === '1') {
5808 - $should_sync = true;
5809 - } else if ($post_type === 'product' && class_exists('WooCommerce')) {
5810 - // WooCommerce products - check if WooCommerce integration is enabled
5811 - $options = get_option('mxchat_options', array());
5812 - if (isset($options['enable_woocommerce_integration']) &&
5813 - ($options['enable_woocommerce_integration'] === '1' || $options['enable_woocommerce_integration'] === 'on')) {
5814 - $should_sync = true;
5815 - }
5816 - } else {
5817 - // Check custom post types
5818 - $option_name = 'mxchat_auto_sync_' . $post_type;
5819 - if (get_option($option_name) === '1') {
5820 - $should_sync = true;
5821 - }
5822 - }
5823 -
5824 - if (!$should_sync) {
5825 - return;
5826 - }
5827 -
5828 - // Only process published posts
3018 +
3019 + // Only process published content
5829 3020 if ($post->post_status !== 'publish') {
5830 3021 return;
5831 3022 }
5832 -
5833 - // Check if this post has any ACF fields - if not, no need to re-sync
5834 - $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
5835 - if (empty($acf_fields)) {
5836 - return;
5837 - }
5838 -
5839 - // Use a transient to prevent duplicate processing (post_updated may have already run)
5840 - $transient_key = 'mxchat_acf_synced_' . $post_id;
5841 - if (get_transient($transient_key)) {
5842 - return;
5843 - }
5844 - set_transient($transient_key, true, 60); // Prevent re-processing for 60 seconds
5845 -
5846 - // Re-run the sync with ACF data now available
5847 - // We pass $update=true since this is effectively an update with ACF data
5848 - $this->mxchat_handle_post_update($post_id, $post, true);
5849 -}
5850 -
5851 -public function mxchat_handle_post_update($post_id, $post, $update) {
5852 - // Basic validation checks
5853 - if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5854 - return;
5855 - }
5856 3023
5857 3024 $post_type = $post->post_type;
5858 3025
5859 3026 // Check if sync is enabled for this post type
@@ -5875,495 +3042,80 @@
5875 3042 if (!$should_sync) {
5876 3043 return;
5877 3044 }
5878 3045
5879 - // Check if we have stored the previous status and URL in our transients
5880 - $previous_status_key = 'mxchat_prev_status_' . $post_id;
5881 - $previous_status = get_transient($previous_status_key);
3046 + // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
3047 + $title = get_the_title($post_id);
3048 + $content = get_post_field('post_content', $post_id);
5882 3049
5883 - $previous_url_key = 'mxchat_prev_url_' . $post_id;
5884 - $previous_url = get_transient($previous_url_key);
3050 + // Apply WordPress content filters to get properly formatted content
3051 + $content = apply_filters('the_content', $content);
5885 3052
5886 - // If the post was previously published but is now not published, remove from knowledge base
5887 - if ($previous_status === 'publish' && $post->post_status !== 'publish') {
5888 - // Use the stored URL from when it was published, or fall back to current permalink
5889 - $source_url = $previous_url ?: get_permalink($post_id);
5890 -
5891 - // mxchat_handle_status_transition already deleted for this post earlier in this
5892 - // request (it fires first inside wp_insert_post); skip the redundant round-trip.
5893 - if ($source_url && empty($this->transition_deleted_posts[$post_id])) {
5894 - // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
5895 - MxChat_Utils::delete_chunks_for_url($source_url, 'default');
5896 - }
5897 -
5898 - // Clean up the transients and exit early
5899 - delete_transient($previous_status_key);
5900 - delete_transient($previous_url_key);
5901 - return;
5902 - }
5903 -
5904 - // Slug/permalink rename while still published: delete the old vectors before upserting new ones.
5905 - // Without this, md5(old_url) vectors (base + chunks) would be orphaned under the stale URL.
5906 - if ($post->post_status === 'publish' && !empty($previous_url)) {
5907 - $current_url = get_permalink($post_id);
5908 - if ($current_url && $current_url !== $previous_url) {
5909 - MxChat_Utils::delete_chunks_for_url($previous_url, 'default');
5910 - }
5911 - }
5912 -
5913 - // Store the current status for next time (if this is an update)
5914 - if ($update) {
5915 - set_transient($previous_status_key, $post->post_status, DAY_IN_SECONDS);
5916 -
5917 - // If the post is currently published, also store its URL
5918 - if ($post->post_status === 'publish') {
5919 - $current_url = get_permalink($post_id);
5920 - set_transient($previous_url_key, $current_url, DAY_IN_SECONDS);
5921 - }
5922 - }
3053 + // Strip tags but preserve structure
3054 + $content = wp_strip_all_tags($content);
5923 3055
5924 - // Only process currently published content for adding/updating.
5925 - // transition_indexed_posts: mxchat_handle_status_transition's arrival edge may have
5926 - // already indexed this post earlier in this request (editor publishes fire
5927 - // transition_post_status first, then post_updated) — skip the duplicate embed.
5928 - // Consume-once: the flag is cleared when honoured, so a LATER save of the same
5929 - // post in one long-running process (WP-CLI scripts, importers) re-indexes normally.
5930 - if ($post->post_status === 'publish') {
5931 - if (!empty($this->transition_indexed_posts[$post_id])) {
5932 - unset($this->transition_indexed_posts[$post_id]);
5933 - } else {
5934 - $this->mxchat_index_published_post($post_id, $post);
3056 + // Combine title and content
3057 + $final_content = $title . "\n\n" . $content;
3058 +
3059 + // For custom post types like job_listing, include additional fields
3060 + if ($post_type === 'job_listing') {
3061 + // Add job-specific meta if available
3062 + $job_location = get_post_meta($post_id, '_job_location', true);
3063 + if (!empty($job_location)) {
3064 + $final_content .= "\n\nLocation: " . $job_location;
5935 3065 }
5936 - }
5937 -
5938 - // Clean up the stored previous status if not used above
5939 - if ($previous_status !== 'publish' || $post->post_status === 'publish') {
5940 - delete_transient($previous_status_key);
5941 - delete_transient($previous_url_key);
5942 - }
5943 -}
5944 -
5945 -/**
5946 - * Index a published post into the knowledge base: preprocessing filter, content
5947 - * assembly (title/excerpt/body), WooCommerce product enrichment, job_listing meta,
5948 - * ACF fields (+ optional PDF extraction), whitelisted custom meta, embedding and
5949 - * upsert, then tag-based role restriction.
5950 - *
5951 - * Shared by the post_updated auto-sync path (mxchat_handle_post_update) and the
5952 - * transition_post_status arrival edge (mxchat_handle_status_transition), so
5953 - * scheduled publishes (wp_publish_post) and direct status=publish inserts index
5954 - * identically to editor saves (plan 3055e1). Pure extraction of the former
5955 - * publish branch — body indentation retained to keep the diff reviewable.
5956 - */
5957 -private function mxchat_index_published_post($post_id, $post) {
5958 - $post_type = $post->post_type;
5959 -
5960 - // Get the source URL
5961 - $source_url = get_permalink($post_id);
5962 -
5963 - /**
5964 - * Allow developers to modify post data before processing into the knowledge base.
5965 - * Same filter and signature as the manual bulk-import path
5966 - * (ajax_mxchat_process_selected_content), so a callback registered once covers
5967 - * every indexing route. Purely additive — zero behaviour change when unhooked.
5968 - * Auto-sync runs under the 'default' bot context, matching the rest of this
5969 - * function.
5970 - *
5971 - * @param WP_Post $post The post about to be indexed.
5972 - * @param string $bot_id Bot context ('default' on auto-sync).
5973 - */
5974 - $post = apply_filters('mxchat_before_process_post', $post, 'default');
5975 - if (!($post instanceof WP_Post)) {
5976 - $post = get_post($post_id); // defend against a bad callback return
5977 - }
5978 -
5979 - // Get content with proper formatting (matching ajax_mxchat_process_selected_content),
5980 - // reading from the FILTERED post object — not re-fetched by ID, which would discard it
5981 - $title = get_the_title($post);
5982 - $content = get_post_field('post_content', $post);
5983 - $excerpt = get_post_field('post_excerpt', $post);
5984 -
5985 - // Remove shortcode tags but preserve content inside them
5986 - $content = $this->strip_shortcode_tags_preserve_content($content);
5987 - $excerpt = $this->strip_shortcode_tags_preserve_content($excerpt);
5988 -
5989 - // Strip tags but preserve structure (don't use 'the_content' filter as it may re-add shortcodes)
5990 - $content = wp_strip_all_tags($content);
5991 -
5992 - // Combine title, short description (if exists), and content
5993 - $final_content = $title . "\n\n";
5994 -
5995 - // Add short description if it exists (WooCommerce products use post_excerpt for short description)
5996 - if (!empty($excerpt)) {
5997 - $final_content .= "Short Description: " . wp_strip_all_tags($excerpt) . "\n\n";
5998 - }
5999 -
6000 - $final_content .= $content;
6001 -
6002 - // For WooCommerce products, include pricing and product details
6003 - if ($post_type === 'product' && class_exists('WooCommerce')) {
6004 - $product = wc_get_product($post_id);
6005 -
6006 - if ($product) {
6007 - // Get pricing information
6008 - $regular_price = $product->get_regular_price();
6009 - $sale_price = $product->get_sale_price();
6010 - $price = $product->get_price();
6011 - $sku = $product->get_sku();
6012 -
6013 - // Get currency symbol
6014 - $currency_symbol = get_woocommerce_currency_symbol();
6015 -
6016 - // Add pricing information
6017 - $final_content .= "\n";
6018 - if (!empty($regular_price)) {
6019 - $final_content .= "Price: " . $currency_symbol . $regular_price . "\n";
6020 - } elseif (!empty($price)) {
6021 - $final_content .= "Price: " . $currency_symbol . $price . "\n";
6022 - }
6023 -
6024 - if (!empty($sale_price) && $sale_price !== $regular_price) {
6025 - $final_content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
6026 - }
6027 -
6028 - // Handle variable products - show price range
6029 - if ($product->is_type('variable')) {
6030 - $min_price = $product->get_variation_price('min');
6031 - $max_price = $product->get_variation_price('max');
6032 - if ($min_price !== $max_price) {
6033 - $final_content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
6034 - }
6035 - }
6036 -
6037 - if (!empty($sku)) {
6038 - $final_content .= "SKU: " . $sku . "\n";
6039 - }
6040 -
6041 - // Get product categories
6042 - $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
6043 - if (!empty($categories) && !is_wp_error($categories)) {
6044 - $final_content .= "Categories: " . implode(', ', $categories) . "\n";
6045 - }
3066 +
3067 + // Get job type terms
3068 + $job_types = get_the_terms($post_id, 'job_listing_type');
3069 + if (!empty($job_types) && !is_wp_error($job_types)) {
3070 + $types = array();
3071 + foreach ($job_types as $type) {
3072 + $types[] = $type->name;
6046 3073 }
3074 + $final_content .= "\n\nJob Type: " . implode(', ', $types);
6047 3075 }
6048 -
6049 - // For custom post types like job_listing, include additional fields
6050 - if ($post_type === 'job_listing') {
6051 - // Add job-specific meta if available
6052 - $job_location = get_post_meta($post_id, '_job_location', true);
6053 - if (!empty($job_location)) {
6054 - $final_content .= "\n\nLocation: " . $job_location;
6055 - }
6056 -
6057 - // Get job type terms
6058 - $job_types = get_the_terms($post_id, 'job_listing_type');
6059 - if (!empty($job_types) && !is_wp_error($job_types)) {
6060 - $types = array();
6061 - foreach ($job_types as $type) {
6062 - $types[] = $type->name;
6063 - }
6064 - $final_content .= "\n\nJob Type: " . implode(', ', $types);
6065 - }
6066 -
6067 - // Get company name if available
6068 - $company_name = get_post_meta($post_id, '_company_name', true);
6069 - if (!empty($company_name)) {
6070 - $final_content .= "\n\nCompany: " . $company_name;
6071 - }
6072 - }
6073 -
6074 - // ADD ACF FIELDS SUPPORT (matches ajax_mxchat_process_selected_content behavior)
6075 - $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
6076 - if (!empty($acf_fields)) {
6077 - $acf_content_parts = array();
6078 - $pdf_attachment_ids = array();
6079 -
6080 - foreach ($acf_fields as $field_name => $field_value) {
6081 - $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
6082 - if (!empty($formatted_value)) {
6083 - // Convert field name to readable label
6084 - $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name));
6085 - $acf_content_parts[] = $field_label . ": " . $formatted_value;
6086 - }
6087 -
6088 - $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
6089 - }
6090 -
6091 - if (!empty($acf_content_parts)) {
6092 - $final_content .= "\n\n" . implode("\n", $acf_content_parts);
6093 - }
6094 -
6095 - // Gate the auto-sync PDF-extraction loop behind an opt-in option.
6096 - // Mirrors the per-batch checkbox the manual content selector has; the
6097 - // 25 MB size cap lives in the shared extractor so it applies in both
6098 - // paths regardless. Default OFF — re-parsing every ACF PDF on every
6099 - // editor save is expensive and most sites don't want it.
6100 - $autosync_extract_acf_pdfs = get_option('mxchat_auto_sync_acf_pdfs', '0') === '1';
6101 - if ($autosync_extract_acf_pdfs && !empty($pdf_attachment_ids)) {
6102 - $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
6103 - $pdf_sections = array();
6104 - foreach ($pdf_attachment_ids as $att_id) {
6105 - $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
6106 - if (!empty($pdf_text)) {
6107 - $pdf_title = get_the_title($att_id);
6108 - $pdf_url = wp_get_attachment_url($att_id);
6109 - $header = 'PDF Attachment';
6110 - if (!empty($pdf_title)) {
6111 - $header .= ': ' . $pdf_title;
6112 - }
6113 - if (!empty($pdf_url)) {
6114 - $header .= ' (' . $pdf_url . ')';
6115 - }
6116 - $pdf_sections[] = $header . "\n" . $pdf_text;
6117 - }
6118 - }
6119 - if (!empty($pdf_sections)) {
6120 - $final_content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
6121 - }
6122 - }
6123 - }
6124 -
6125 - // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
6126 - $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
6127 - if (!empty($custom_meta)) {
6128 - $meta_content_parts = array();
6129 -
6130 - foreach ($custom_meta as $meta_key => $meta_value) {
6131 - // Convert meta key to readable label
6132 - $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
6133 - $meta_content_parts[] = $meta_label . ": " . $meta_value;
6134 - }
6135 -
6136 - if (!empty($meta_content_parts)) {
6137 - $final_content .= "\n\n" . implode("\n", $meta_content_parts);
6138 - }
6139 - }
6140 -
6141 - // Embedding decision — custom-provider-aware. Gating on a cloud API key
6142 - // here silently killed auto-sync on keyless custom-embeddings sites,
6143 - // because generate_embedding() routes custom FIRST and never needs the
6144 - // key (plan cbd5fd). Silent-return shape preserved.
6145 - $preflight = MxChat_Utils::embedding_preflight(get_option('mxchat_options'));
6146 - if (!$preflight['ok']) {
6147 - return;
6148 - }
6149 - $api_key = $preflight['api_key'];
6150 -
6151 - // Use the centralized utility function for storage
6152 - $result = MxChat_Utils::submit_content_to_db(
6153 - $final_content,
6154 - $source_url,
6155 - $api_key,
6156 - md5($source_url) // Vector ID for Pinecone
6157 - );
6158 3076
6159 - // After successful storage, apply role restriction based on tags
6160 - if (!is_wp_error($result)) {
6161 - $this->apply_role_restriction_to_post($post_id, $source_url);
3077 + // Get company name if available
3078 + $company_name = get_post_meta($post_id, '_company_name', true);
3079 + if (!empty($company_name)) {
3080 + $final_content .= "\n\nCompany: " . $company_name;
6162 3081 }
6163 -}
6164 -
6165 -/**
6166 - * Store the post status and URL before update to detect status transitions
6167 - * This runs before the post is actually updated in the database
6168 - */
6169 -public function mxchat_store_pre_update_status($post_id, $data) {
6170 - // Get the current post from database (before update)
6171 - $current_post = get_post($post_id);
3082 + }
6172 3083
6173 - if ($current_post) {
6174 - // Store the current status temporarily
6175 - $status_key = 'mxchat_prev_status_' . $post_id;
6176 - set_transient($status_key, $current_post->post_status, HOUR_IN_SECONDS);
6177 -
6178 - // If the post is currently published, also store its URL
6179 - if ($current_post->post_status === 'publish') {
6180 - $url_key = 'mxchat_prev_url_' . $post_id;
6181 - $current_url = get_permalink($post_id);
6182 - set_transient($url_key, $current_url, HOUR_IN_SECONDS);
6183 - }
3084 + // Get the source URL
3085 + $source_url = get_permalink($post_id);
3086 +
3087 + // Get API key with proper model detection
3088 + $options = get_option('mxchat_options');
3089 + $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
3090 +
3091 + if (strpos($selected_model, 'voyage') === 0) {
3092 + $api_key = $options['voyage_api_key'] ?? '';
3093 + } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
3094 + $api_key = $options['gemini_api_key'] ?? '';
3095 + } else {
3096 + $api_key = $options['api_key'] ?? '';
6184 3097 }
6185 -}
6186 -
6187 -/**
6188 - * Whether auto-sync is enabled for a post type (mirrors the checks used by the
6189 - * update/delete handlers; kept as one helper so new call sites cannot drift).
6190 - */
6191 -private function mxchat_is_auto_sync_enabled($post_type) {
6192 - if ($post_type === 'post') {
6193 - return get_option('mxchat_auto_sync_posts') === '1';
6194 - }
6195 - if ($post_type === 'page') {
6196 - return get_option('mxchat_auto_sync_pages') === '1';
6197 - }
6198 - return get_option('mxchat_auto_sync_' . $post_type) === '1';
6199 -}
6200 -
6201 -/**
6202 - * Remove a post's vectors the moment it leaves 'publish', using the authoritative
6203 - * old status core passes to transition_post_status — no transient involved (plan 816fb1).
6204 - *
6205 - * Covers status changes that never route through wp_update_post (scheduled-expiry
6206 - * plugins and others that flip post_status directly and call wp_transition_post_status),
6207 - * where neither pre_post_update nor post_updated fires and the old detection missed.
6208 - */
6209 -public function mxchat_handle_status_transition($new_status, $old_status, $post) {
6210 - if (!($post instanceof WP_Post) || wp_is_post_revision($post->ID)) {
3098 +
3099 + if (empty($api_key)) {
3100 + //error_log('MxChat Auto-sync: No API key configured for embedding model');
6211 3101 return;
6212 3102 }
6213 -
6214 - // Arrival edge (plan 3055e1): a post BECOMING published is indexed here, because
6215 - // wp_publish_post() — the path scheduled posts take via check_and_publish_future_post —
6216 - // and direct wp_insert_post(status=publish) creates never fire post_updated, so the
6217 - // auto-sync ADD path alone misses them. Editor publishes also pass through here;
6218 - // the transition_indexed_posts guard keeps mxchat_handle_post_update from embedding
6219 - // a second time in the same request.
6220 - if ($new_status === 'publish' && $old_status !== 'publish') {
6221 - if ($this->mxchat_is_auto_sync_enabled($post->post_type)) {
6222 - $this->mxchat_index_published_post($post->ID, $post);
6223 - $this->transition_indexed_posts[$post->ID] = true;
6224 - }
6225 - return;
3103 +
3104 + // Use the centralized utility function for storage
3105 + $result = MxChat_Utils::submit_content_to_db(
3106 + $final_content,
3107 + $source_url,
3108 + $api_key,
3109 + md5($source_url) // Vector ID for Pinecone
3110 + );
3111 +
3112 + if (is_wp_error($result)) {
3113 + //error_log('MxChat Auto-sync failed for post ' . $post_id . ': ' . $result->get_error_message());
6226 3114 }
6227 -
6228 - // Only the publish -> not-publish edge matters here.
6229 - if ($old_status !== 'publish' || $new_status === 'publish') {
6230 - return;
6231 - }
6232 - // Trash is handled by mxchat_handle_post_delete (wp_trash_post) with pre-trash URL
6233 - // resolution; skip to avoid a second network round-trip per trash.
6234 - if ($new_status === 'trash') {
6235 - return;
6236 - }
6237 - if (!$this->mxchat_is_auto_sync_enabled($post->post_type)) {
6238 - return;
6239 - }
6240 -
6241 - $urls = array();
6242 -
6243 - // The DB may already hold the new status when this fires, so get_permalink() on the
6244 - // live post could build a draft-style URL whose md5 misses the stored vector IDs.
6245 - // Reconstruct the published permalink from a clone instead.
6246 - $published_clone = clone $post;
6247 - $published_clone->post_status = 'publish';
6248 - $published_url = get_permalink($published_clone);
6249 - if ($published_url) {
6250 - $urls[] = $published_url;
6251 - }
6252 -
6253 - // Honour the pre-update capture when present (covers a slug change in the same save).
6254 - $previous_url = get_transient('mxchat_prev_url_' . $post->ID);
6255 - if (!empty($previous_url)) {
6256 - $urls[] = $previous_url;
6257 - }
6258 -
6259 - foreach (array_unique($urls) as $url) {
6260 - MxChat_Utils::delete_chunks_for_url($url, 'default');
6261 - }
6262 -
6263 - if (!empty($urls)) {
6264 - $this->transition_deleted_posts[$post->ID] = true;
6265 - }
6266 3115 }
6267 3116
6268 -/**
6269 - * WP-CLI: remove knowledge-base entries left behind by posts that were unpublished,
6270 - * trashed, or made private before the transition_post_status handler existed.
6271 - *
6272 - * Walks every auto-synced post type's non-published posts, reconstructs each one's
6273 - * published-era permalink, and deletes its vectors (routes to Pinecone or the WP table).
6274 - * Deletion is idempotent, so never-indexed posts are a cheap no-op.
6275 - *
6276 - * ## OPTIONS
6277 - *
6278 - * [--dry-run]
6279 - * : Report what would be removed without deleting anything.
6280 - *
6281 - * ## EXAMPLES
6282 - *
6283 - * wp mxchat prune-unpublished --dry-run
6284 - * wp mxchat prune-unpublished
6285 - */
6286 -public function cli_prune_unpublished($args, $assoc_args) {
6287 - global $wpdb;
6288 - $dry_run = !empty($assoc_args['dry-run']);
6289 - $table = $wpdb->prefix . 'mxchat_system_prompt_content';
6290 3117
6291 - $candidate_types = array_merge(array('post', 'page'), array_values(get_post_types(array('_builtin' => false), 'names')));
6292 - $synced_types = array();
6293 - foreach ($candidate_types as $type) {
6294 - if ($this->mxchat_is_auto_sync_enabled($type)) {
6295 - $synced_types[] = $type;
6296 - }
6297 - }
6298 - if (empty($synced_types)) {
6299 - WP_CLI::success('No post types have auto-sync enabled; nothing to prune.');
6300 - return;
6301 - }
6302 -
6303 - $scanned = 0;
6304 - $pruned = 0;
6305 - $paged = 1;
6306 - do {
6307 - $query = new WP_Query(array(
6308 - 'post_type' => $synced_types,
6309 - 'post_status' => array('draft', 'pending', 'private', 'future', 'trash'),
6310 - 'posts_per_page' => 100,
6311 - 'paged' => $paged,
6312 - 'fields' => 'ids',
6313 - ));
6314 - foreach ($query->posts as $post_id) {
6315 - $post = get_post($post_id);
6316 - if (!$post) {
6317 - continue;
6318 - }
6319 - $scanned++;
6320 -
6321 - // Rebuild the permalink the post had while published: publish-status clone,
6322 - // with wp_trash_post's __trashed slug suffix stripped for trashed posts.
6323 - $clone = clone $post;
6324 - $clone->post_status = 'publish';
6325 - if (substr($clone->post_name, -9) === '__trashed') {
6326 - $clone->post_name = substr($clone->post_name, 0, -9);
6327 - }
6328 - $url = get_permalink($clone);
6329 - if (!$url) {
6330 - continue;
6331 - }
6332 -
6333 - // Local-table row count is exact in WordPress-DB mode; in Pinecone mode it
6334 - // reads 0 but the delete below still routes to Pinecone and is idempotent.
6335 - $local_rows = (int) $wpdb->get_var($wpdb->prepare(
6336 - "SELECT COUNT(*) FROM {$table} WHERE source_url = %s", $url
6337 - ));
6338 -
6339 - if ($dry_run) {
6340 - if ($local_rows > 0) {
6341 - WP_CLI::log(sprintf('Would remove %d row(s): %s (post %d, %s)', $local_rows, $url, $post_id, $post->post_status));
6342 - $pruned += $local_rows;
6343 - }
6344 - continue;
6345 - }
6346 -
6347 - MxChat_Utils::delete_chunks_for_url($url, 'default');
6348 - if ($local_rows > 0) {
6349 - WP_CLI::log(sprintf('Removed %d row(s): %s (post %d, %s)', $local_rows, $url, $post_id, $post->post_status));
6350 - $pruned += $local_rows;
6351 - }
6352 - }
6353 - $more = $paged < $query->max_num_pages;
6354 - $paged++;
6355 - } while ($more);
6356 -
6357 - WP_CLI::success(sprintf(
6358 - '%s %d local knowledge row(s) across %d non-published post(s) scanned.%s',
6359 - $dry_run ? 'Would remove' : 'Removed',
6360 - $pruned,
6361 - $scanned,
6362 - ' (Pinecone-mode deletions are not counted locally.)'
6363 - ));
6364 -}
6365 -
6366 3118 public function mxchat_handle_post_delete($post_id) {
6367 3119 // Get post data before it's deleted
6368 3120 $post = get_post($post_id);
6369 3121
@@ -6393,51 +3145,85 @@
6393 3145 if (!$should_sync) {
6394 3146 return;
6395 3147 }
6396 3148
6397 - // Resolve the pre-trash URL. wp_trash_post renames the slug with "__trashed" before firing
6398 - // this hook, so get_permalink() here would return the trashed URL and md5() would miss the
6399 - // real vector IDs stored under the original URL.
6400 - $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
3149 + // Get the URL before post is deleted
3150 + $source_url = get_permalink($post_id);
6401 3151 if (!$source_url) {
6402 - //error_log('MXChat: Failed to resolve source URL for post ' . $post_id);
3152 + //error_log('MXChat: Failed to get permalink for post ' . $post_id);
6403 3153 return;
6404 3154 }
6405 3155
6406 - // Use chunk-aware deletion (handles both chunked and non-chunked content)
6407 - $delete_result = MxChat_Utils::delete_chunks_for_url($source_url, 'default');
3156 + // Check if Pinecone is enabled
3157 + $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
3158 + $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6408 3159
6409 - if (is_wp_error($delete_result)) {
6410 - //error_log('MXChat: Chunk-aware deletion failed for URL: ' . $source_url . ' - ' . $delete_result->get_error_message());
3160 + if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3161 + // Delete from Pinecone
3162 + $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
3163 + } else {
3164 + // Delete from WordPress DB
3165 + global $wpdb;
3166 + $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3167 +
3168 + $result = $wpdb->delete(
3169 + $table_name,
3170 + array('source_url' => $source_url),
3171 + array('%s')
3172 + );
3173 +
3174 + if ($result === false) {
3175 + //error_log('MXChat: WordPress DB deletion failed for URL: ' . $source_url);
3176 + }
6411 3177 }
3178 +}
6412 3179
6413 - delete_transient('mxchat_prev_url_' . $post_id);
6414 - delete_transient('mxchat_prev_status_' . $post_id);
6415 -}
6416 3180
6417 -/**
6418 - * Resolve the source URL for a post being trashed/deleted.
6419 - *
6420 - * Why: wp_trash_post appends "__trashed" to the slug before the wp_trash_post action fires, so
6421 - * get_permalink() returns a URL whose md5() won't match the vector IDs stored in Pinecone or
6422 - * the source_url rows in the WP DB. Prefer the URL captured by mxchat_store_pre_update_status
6423 - * (runs on pre_post_update, before the rename); fall back to stripping the __trashed suffix.
6424 - */
6425 -private function mxchat_resolve_pre_trash_url($post_id) {
6426 - $previous_url = get_transient('mxchat_prev_url_' . $post_id);
6427 - if (!empty($previous_url)) {
6428 - return $previous_url;
6429 - }
3181 + /**
3182 + * Deletes data from Pinecone using a source URL
3183 + */
3184 + public function mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options) {
3185 + $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
3186 + $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
6430 3187
6431 - $current = get_permalink($post_id);
6432 - if (!$current) {
6433 - return '';
6434 - }
6435 - return preg_replace('#__trashed(/?)$#', '$1', $current);
6436 -}
3188 + if (empty($host) || empty($api_key)) {
3189 + //error_log('MXChat: Pinecone deletion failed - missing configuration');
3190 + return false;
3191 + }
6437 3192
3193 + $api_endpoint = "https://{$host}/vectors/delete";
3194 + $vector_id = md5($source_url);
6438 3195
3196 + $request_body = array(
3197 + 'ids' => array($vector_id)
3198 + );
6439 3199
3200 + $response = wp_remote_post($api_endpoint, array(
3201 + 'headers' => array(
3202 + 'Api-Key' => $api_key,
3203 + 'accept' => 'application/json',
3204 + 'content-type' => 'application/json'
3205 + ),
3206 + 'body' => wp_json_encode($request_body),
3207 + 'timeout' => 30
3208 + ));
3209 +
3210 + if (is_wp_error($response)) {
3211 + //error_log('MXChat: Pinecone deletion error - ' . $response->get_error_message());
3212 + return false;
3213 + }
3214 +
3215 + $response_code = wp_remote_retrieve_response_code($response);
3216 + if ($response_code !== 200) {
3217 + //error_log('MXChat: Pinecone deletion failed with status ' . $response_code);
3218 + return false;
3219 + }
3220 +
3221 + return true;
3222 + }
3223 +
3224 +
3225 +
6440 3226 public function mxchat_handle_product_change($post_id, $post, $update) {
6441 3227 if ($post->post_type !== 'product') {
6442 3228 return;
6443 3229 }
@@ -6459,12 +3245,11 @@
6459 3245 if (!isset($this->options['enable_woocommerce_integration']) ||
6460 3246 !in_array($this->options['enable_woocommerce_integration'], ['1', 'on'])) {
6461 3247 return;
6462 3248 }
6463 -
3249 +
6464 3250 $source_url = get_permalink($product->get_id());
6465 - $product_id = $product->get_id();
6466 -
3251 +
6467 3252 // Build product content
6468 3253 $title = $product->get_name();
6469 3254 $description = $product->get_description();
6470 3255 $short_description = $product->get_short_description();
@@ -6469,97 +3254,48 @@
6469 3254 $description = $product->get_description();
6470 3255 $short_description = $product->get_short_description();
6471 3256 $regular_price = $product->get_regular_price();
6472 3257 $sale_price = $product->get_sale_price();
6473 - $price = $product->get_price();
6474 3258 $sku = $product->get_sku();
6475 -
6476 - // Get currency symbol
6477 - $currency_symbol = get_woocommerce_currency_symbol();
6478 -
3259 +
6479 3260 // Format content consistently
6480 3261 $content = $title . "\n\n";
6481 -
6482 - if (!empty($short_description)) {
6483 - $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
6484 - }
6485 -
3262 +
6486 3263 if (!empty($description)) {
6487 3264 $content .= wp_strip_all_tags($description) . "\n\n";
6488 3265 }
6489 -
6490 - // Add pricing information
6491 - if (!empty($regular_price)) {
6492 - $content .= "Price: " . $currency_symbol . $regular_price . "\n";
6493 - } elseif (!empty($price)) {
6494 - $content .= "Price: " . $currency_symbol . $price . "\n";
3266 +
3267 + if (!empty($short_description)) {
3268 + $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
6495 3269 }
6496 -
6497 - if (!empty($sale_price) && $sale_price !== $regular_price) {
6498 - $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
3270 +
3271 + $content .= "Price: $" . $regular_price . "\n";
3272 +
3273 + if (!empty($sale_price)) {
3274 + $content .= "Sale Price: $" . $sale_price . "\n";
6499 3275 }
6500 -
6501 - // Handle variable products - show price range
6502 - if ($product->is_type('variable')) {
6503 - $min_price = $product->get_variation_price('min');
6504 - $max_price = $product->get_variation_price('max');
6505 - if ($min_price !== $max_price) {
6506 - $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
6507 - }
6508 - }
6509 -
3276 +
6510 3277 if (!empty($sku)) {
6511 3278 $content .= "SKU: " . $sku . "\n";
6512 3279 }
6513 -
6514 - // Get product categories
6515 - $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
6516 - if (!empty($categories) && !is_wp_error($categories)) {
6517 - $content .= "Categories: " . implode(', ', $categories) . "\n";
3280 +
3281 + // Get API key with proper model detection
3282 + $options = get_option('mxchat_options');
3283 + $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
3284 +
3285 + if (strpos($selected_model, 'voyage') === 0) {
3286 + $api_key = $options['voyage_api_key'] ?? '';
3287 + } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
3288 + $api_key = $options['gemini_api_key'] ?? '';
3289 + } else {
3290 + $api_key = $options['api_key'] ?? '';
6518 3291 }
6519 -
6520 - // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
6521 - $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
6522 - if (!empty($custom_tabs) && is_array($custom_tabs)) {
6523 - foreach ($custom_tabs as $tab) {
6524 - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
6525 - $tab_content = isset($tab['content']) ? $tab['content'] : '';
6526 -
6527 - if (!empty($tab_title) && !empty($tab_content)) {
6528 - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
6529 - }
6530 - }
6531 - }
6532 -
6533 - // Also check for reusable/saved tabs applied to this product
6534 - $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
6535 - if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
6536 - // Get the saved tabs option
6537 - $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
6538 - if (!empty($saved_tabs) && is_array($saved_tabs)) {
6539 - foreach ($applied_saved_tabs as $saved_tab_id) {
6540 - if (isset($saved_tabs[$saved_tab_id])) {
6541 - $tab = $saved_tabs[$saved_tab_id];
6542 - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
6543 - $tab_content = isset($tab['content']) ? $tab['content'] : '';
6544 -
6545 - if (!empty($tab_title) && !empty($tab_content)) {
6546 - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
6547 - }
6548 - }
6549 - }
6550 - }
6551 - }
6552 -
6553 - // Embedding decision — custom-provider-aware (plan cbd5fd); silent-return
6554 - // shape preserved.
6555 - $preflight = MxChat_Utils::embedding_preflight(get_option('mxchat_options'));
6556 - if (!$preflight['ok']) {
6557 - //error_log('MxChat Auto-sync: embedding pre-flight failed: ' . $preflight['reason']);
3292 +
3293 + if (empty($api_key)) {
3294 + //error_log('MxChat Auto-sync: No API key configured for embedding model');
6558 3295 return;
6559 3296 }
6560 - $api_key = $preflight['api_key'];
6561 -
3297 +
6562 3298 // Use the centralized utility function for storage
6563 3299 $result = MxChat_Utils::submit_content_to_db(
6564 3300 $content,
6565 3301 $source_url,
@@ -6565,16 +3301,11 @@
6565 3301 $source_url,
6566 3302 $api_key,
6567 3303 md5($source_url) // Vector ID for Pinecone
6568 3304 );
6569 -
6570 - // After successful storage, apply role restriction based on tags
6571 - if (!is_wp_error($result)) {
6572 - $this->apply_role_restriction_to_post($product_id, $source_url);
6573 - }
6574 -
3305 +
6575 3306 if (is_wp_error($result)) {
6576 - //error_log('MxChat WooCommerce sync failed for product ' . $product_id . ': ' . $result->get_error_message());
3307 + //error_log('MxChat WooCommerce sync failed for product ' . $product->get_id() . ': ' . $result->get_error_message());
6577 3308 }
6578 3309 }
6579 3310
6580 3311 public function mxchat_handle_product_delete($post_id) {
@@ -6581,1123 +3312,30 @@
6581 3312 if (get_post_type($post_id) !== 'product') {
6582 3313 return;
6583 3314 }
6584 3315
6585 - $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
6586 - if (!$source_url) {
6587 - return;
6588 - }
3316 + $source_url = get_permalink($post_id);
6589 3317
6590 - // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
6591 - MxChat_Utils::delete_chunks_for_url($source_url, 'default');
6592 -
6593 - delete_transient('mxchat_prev_url_' . $post_id);
6594 - delete_transient('mxchat_prev_status_' . $post_id);
6595 -}
6596 -
6597 -/**
6598 - * Handle individual Pinecone content deletion
6599 - */
6600 -public function mxchat_handle_pinecone_prompt_delete() {
6601 - // Check permissions
6602 - if (!current_user_can('manage_options')) {
6603 - wp_die(esc_html__('You do not have sufficient permissions.', 'mxchat'));
6604 - }
6605 -
6606 - // Verify nonce
6607 - if (!isset($_GET['_wpnonce']) || !wp_verify_nonce($_GET['_wpnonce'], 'mxchat_delete_pinecone_prompt_nonce')) {
6608 - wp_die(esc_html__('Security check failed.', 'mxchat'));
6609 - }
6610 -
6611 - $vector_id = isset($_GET['vector_id']) ? sanitize_text_field($_GET['vector_id']) : '';
6612 -
6613 - if (empty($vector_id)) {
6614 - set_transient('mxchat_admin_notice_error',
6615 - esc_html__('Invalid vector ID.', 'mxchat'),
6616 - 30
6617 - );
6618 - wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
6619 - exit;
6620 - }
6621 -
6622 - // Get Pinecone settings
3318 + // Check if Pinecone is enabled
6623 3319 $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6624 3320 $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6625 -
6626 - if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
6627 - set_transient('mxchat_admin_notice_error',
6628 - esc_html__('Pinecone is not properly configured.', 'mxchat'),
6629 - 30
6630 - );
6631 - wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
6632 - exit;
6633 - }
6634 -
6635 - // Delete from Pinecone
6636 - $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6637 - $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
6638 - $vector_id,
6639 - $pinecone_options['mxchat_pinecone_api_key'],
6640 - $pinecone_options['mxchat_pinecone_host']
6641 - );
6642 -
6643 - if ($result['success']) {
6644 - // No cache clearing needed since we removed caching
6645 - set_transient('mxchat_admin_notice_success',
6646 - esc_html__('Entry deleted successfully from Pinecone.', 'mxchat'),
6647 - 30
6648 - );
6649 - } else {
6650 - set_transient('mxchat_admin_notice_error',
6651 - esc_html__('Failed to delete entry: ', 'mxchat') . $result['message'],
6652 - 30
6653 - );
6654 - }
6655 -
6656 - wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
6657 - exit;
6658 -}
6659 -/**
6660 - * Handle individual Pinecone content deletion via AJAX
6661 - */
6662 -public function ajax_mxchat_delete_pinecone_prompt() {
6663 - // Verify nonce and permissions
6664 - if (!check_ajax_referer('mxchat_delete_pinecone_prompt_nonce', 'nonce', false)) {
6665 - wp_send_json_error('Invalid nonce');
6666 - exit;
6667 - }
6668 -
6669 - if (!current_user_can('manage_options')) {
6670 - wp_send_json_error('Unauthorized access');
6671 - exit;
6672 - }
6673 -
6674 - $vector_id = isset($_POST['vector_id']) ? sanitize_text_field($_POST['vector_id']) : '';
6675 - $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6676 -
6677 - if (empty($vector_id)) {
6678 - wp_send_json_error('Missing vector ID');
6679 - exit;
6680 - }
6681 -
6682 - // Get bot-specific Pinecone settings
6683 - $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6684 - $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6685 -
6686 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6687 -
6688 - if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
6689 - wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
6690 - exit;
6691 - }
6692 -
6693 - // Delete from the correct Pinecone index
6694 - $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
6695 - $vector_id,
6696 - $pinecone_options['mxchat_pinecone_api_key'],
6697 - $pinecone_options['mxchat_pinecone_host']
6698 - );
6699 -
6700 - if ($result['success']) {
6701 - // No cache clearing needed since we removed caching
6702 - wp_send_json_success(array(
6703 - 'message' => 'Entry deleted successfully from Pinecone',
6704 - 'vector_id' => $vector_id,
6705 - 'bot_id' => $bot_id
6706 - ));
6707 - } else {
6708 - MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete from Pinecone: ' . $result['message'], array('vector_id' => $vector_id, 'bot_id' => $bot_id));
6709 - wp_send_json_error('Failed to delete from Pinecone: ' . $result['message']);
6710 - }
6711 -
6712 - exit;
6713 -}
6714 3321
6715 -/**
6716 - * Handle deletion of all chunks for a given source URL via AJAX
6717 - * Follows the same pattern as ajax_mxchat_delete_pinecone_prompt
6718 - */
6719 -public function ajax_mxchat_delete_chunks_by_url() {
6720 - // Verify nonce and permissions
6721 - if (!check_ajax_referer('mxchat_delete_chunks_nonce', 'nonce', false)) {
6722 - wp_send_json_error('Invalid nonce');
6723 - exit;
6724 - }
6725 -
6726 - if (!current_user_can('manage_options')) {
6727 - wp_send_json_error('Unauthorized access');
6728 - exit;
6729 - }
6730 -
6731 - $source_url = isset($_POST['source_url']) ? esc_url_raw($_POST['source_url']) : '';
6732 - $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
6733 - $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6734 -
6735 - if (empty($source_url)) {
6736 - wp_send_json_error('Missing source URL');
6737 - exit;
6738 - }
6739 -
6740 - // Generate the base vector ID from the source URL (same as how chunks are created)
6741 - $base_vector_id = md5($source_url);
6742 -
6743 - if ($data_source === 'pinecone') {
6744 - // Get bot-specific Pinecone settings (same as working delete function)
6745 - $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6746 - $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6747 -
6748 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6749 -
6750 - if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
6751 - wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
6752 - exit;
6753 - }
6754 -
6755 - $api_key = $pinecone_options['mxchat_pinecone_api_key'];
6756 - $host = $pinecone_options['mxchat_pinecone_host'];
6757 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
6758 -
6759 - // Collect all vector IDs to delete
6760 - $vectors_to_delete = array();
6761 -
6762 - // Add the original single-vector ID (for non-chunked content)
6763 - $vectors_to_delete[] = $base_vector_id;
6764 -
6765 - // Use Pinecone list API to find all chunk vectors with this prefix
6766 - // NOTE: Pinecone List API is a GET request with query parameters, not POST
6767 - $prefix = $base_vector_id . '_chunk_';
6768 -
6769 - $query_params = array(
6770 - 'prefix' => $prefix,
6771 - 'limit' => 100
6772 - );
6773 -
6774 - if (!empty($namespace)) {
6775 - $query_params['namespace'] = $namespace;
6776 - }
6777 -
6778 - $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
6779 -
6780 - $list_response = wp_remote_get($list_url, array(
6781 - 'headers' => array(
6782 - 'Api-Key' => $api_key,
6783 - 'accept' => 'application/json'
6784 - ),
6785 - 'timeout' => 30
6786 - ));
6787 -
6788 - if (!is_wp_error($list_response)) {
6789 - $list_body_response = wp_remote_retrieve_body($list_response);
6790 - $list_data = json_decode($list_body_response, true);
6791 - if (!empty($list_data['vectors'])) {
6792 - foreach ($list_data['vectors'] as $vector) {
6793 - if (isset($vector['id'])) {
6794 - $vectors_to_delete[] = $vector['id'];
6795 - }
6796 - }
6797 - }
6798 - }
6799 -
6800 - if (empty($vectors_to_delete)) {
6801 - wp_send_json_success(array(
6802 - 'message' => 'No vectors found to delete',
6803 - 'source_url' => $source_url
6804 - ));
6805 - exit;
6806 - }
6807 -
6808 - // Delete all vectors using the same endpoint as the working function
6809 - $delete_url = "https://{$host}/vectors/delete";
6810 -
6811 - $delete_body = array(
6812 - 'ids' => $vectors_to_delete
6813 - );
6814 -
6815 - if (!empty($namespace)) {
6816 - $delete_body['namespace'] = $namespace;
6817 - }
6818 -
6819 - $delete_response = wp_remote_post($delete_url, array(
6820 - 'headers' => array(
6821 - 'Api-Key' => $api_key,
6822 - 'accept' => 'application/json',
6823 - 'content-type' => 'application/json'
6824 - ),
6825 - 'body' => wp_json_encode($delete_body),
6826 - 'timeout' => 30
6827 - ));
6828 -
6829 - if (is_wp_error($delete_response)) {
6830 - MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete chunks from Pinecone: ' . $delete_response->get_error_message(), array('source_url' => $source_url));
6831 - wp_send_json_error('Failed to delete from Pinecone: ' . $delete_response->get_error_message());
6832 - exit;
6833 - }
6834 -
6835 - $response_code = wp_remote_retrieve_response_code($delete_response);
6836 -
6837 - if ($response_code !== 200) {
6838 - MxChat_Admin::mxchat_log_debug('pinecone_error', 'Pinecone API error (HTTP ' . $response_code . ')', array('source_url' => $source_url));
6839 - wp_send_json_error('Pinecone API error (HTTP ' . $response_code . ')');
6840 - exit;
6841 - }
6842 -
6843 - wp_send_json_success(array(
6844 - 'message' => 'All chunks deleted successfully from Pinecone',
6845 - 'source_url' => $source_url,
6846 - 'deleted_count' => count($vectors_to_delete)
6847 - ));
6848 -
3322 + if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3323 + // Delete from Pinecone
3324 + $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
6849 3325 } else {
6850 - // WordPress database deletion
3326 + // Delete from WordPress DB
6851 3327 global $wpdb;
6852 3328 $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6853 -
6854 - $result = $wpdb->delete(
6855 - $table_name,
6856 - array('source_url' => $source_url),
6857 - array('%s')
6858 - );
6859 -
6860 - if ($result === false) {
6861 - MxChat_Admin::mxchat_log_debug('database_error', 'Failed to delete from database: ' . $wpdb->last_error, array('source_url' => $source_url));
6862 - wp_send_json_error('Failed to delete from database: ' . $wpdb->last_error);
6863 - exit;
6864 - }
6865 -
6866 - wp_send_json_success(array(
6867 - 'message' => 'All chunks deleted successfully from database',
6868 - 'source_url' => $source_url,
6869 - 'deleted_count' => $result
6870 - ));
6871 - }
6872 -
6873 - exit;
6874 -}
6875 -
6876 -/**
6877 - * Handle individual WordPress database content deletion via AJAX
6878 - * Mirrors the Pinecone delete handler but for WordPress database entries
6879 - */
6880 -public function ajax_mxchat_delete_wordpress_prompt() {
6881 - // Verify nonce and permissions
6882 - if (!check_ajax_referer('mxchat_delete_wordpress_prompt_nonce', 'nonce', false)) {
6883 - wp_send_json_error('Invalid nonce');
6884 - exit;
6885 - }
6886 -
6887 - if (!current_user_can('manage_options')) {
6888 - wp_send_json_error('Unauthorized access');
6889 - exit;
6890 - }
6891 -
6892 - $entry_id = isset($_POST['entry_id']) ? intval($_POST['entry_id']) : 0;
6893 -
6894 - if (empty($entry_id)) {
6895 - wp_send_json_error('Missing entry ID');
6896 - exit;
6897 - }
6898 -
6899 - global $wpdb;
6900 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6901 -
6902 - // Clear cache for this entry
6903 - wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
6904 -
6905 - // Delete from database
6906 - $result = $wpdb->delete(
6907 - $table_name,
6908 - array('id' => $entry_id),
6909 - array('%d')
6910 - );
6911 -
6912 - if ($result !== false) {
6913 - wp_send_json_success(array(
6914 - 'message' => 'Entry deleted successfully',
6915 - 'entry_id' => $entry_id
6916 - ));
6917 - } else {
6918 - wp_send_json_error('Failed to delete entry from database');
6919 - }
6920 -
6921 - exit;
6922 -}
6923 -
6924 -/**
6925 - * Handle bulk deletion of knowledge entries via AJAX
6926 - * Supports both Pinecone and WordPress database entries
6927 - */
6928 -public function ajax_mxchat_bulk_delete_knowledge() {
6929 - // Verify nonce and permissions
6930 - if (!check_ajax_referer('mxchat_bulk_delete_knowledge_nonce', 'nonce', false)) {
6931 - wp_send_json_error('Invalid nonce');
6932 - exit;
6933 - }
6934 -
6935 - if (!current_user_can('manage_options')) {
6936 - wp_send_json_error('Unauthorized access');
6937 - exit;
6938 - }
6939 -
6940 - $entries = isset($_POST['entries']) ? $_POST['entries'] : array();
6941 - $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6942 -
6943 - if (empty($entries) || !is_array($entries)) {
6944 - wp_send_json_error('No entries provided');
6945 - exit;
6946 - }
6947 -
6948 - // Extend execution time — bulk Pinecone operations can take a while
6949 - if (function_exists('set_time_limit')) {
6950 - set_time_limit(120);
6951 - }
6952 -
6953 - $success_ids = array();
6954 - $failed_ids = array();
6955 - $errors = array();
6956 -
6957 - global $wpdb;
6958 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6959 -
6960 - // Get Pinecone manager for Pinecone deletions
6961 - $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6962 - $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6963 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6964 -
6965 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
6966 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
6967 -
6968 - // =============================================
6969 - // PHASE 1: Collect all Pinecone vector IDs
6970 - // and separate WordPress entries
6971 - // =============================================
6972 - $pinecone_entry_ids = array(); // entry IDs that are pinecone-sourced
6973 - $wordpress_entries = array(); // entries for WordPress DB deletion
6974 - $all_vector_ids = array(); // all pinecone vector IDs to delete in one batch
6975 -
6976 - foreach ($entries as $entry) {
6977 - $entry_id = sanitize_text_field($entry['id'] ?? '');
6978 - $source = sanitize_text_field($entry['source'] ?? 'wordpress');
6979 - $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
6980 - $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
6981 -
6982 - if (empty($entry_id)) {
6983 - continue;
6984 - }
6985 -
6986 - if ($source === 'pinecone') {
6987 - if (!$use_pinecone || empty($api_key)) {
6988 - $failed_ids[] = $entry_id;
6989 - $errors[] = "Pinecone not configured for entry: $entry_id";
6990 - continue;
6991 - }
6992 -
6993 - $pinecone_entry_ids[] = $entry_id;
6994 -
6995 - if ($is_group && !empty($source_url)) {
6996 - // Grouped/chunked entry: collect base ID + chunk IDs via List API
6997 - $base_vector_id = md5($source_url);
6998 - $all_vector_ids[] = $base_vector_id;
6999 -
7000 - $list_url = 'https://' . $host . '/vectors/list?prefix=' . urlencode($base_vector_id . '_chunk_') . '&limit=100';
7001 - $list_response = wp_remote_get($list_url, array(
7002 - 'headers' => array(
7003 - 'Api-Key' => $api_key,
7004 - 'accept' => 'application/json'
7005 - ),
7006 - 'timeout' => 30
7007 - ));
7008 -
7009 - if (!is_wp_error($list_response)) {
7010 - $list_body = json_decode(wp_remote_retrieve_body($list_response), true);
7011 - if (!empty($list_body['vectors']) && is_array($list_body['vectors'])) {
7012 - foreach ($list_body['vectors'] as $vector) {
7013 - if (isset($vector['id'])) {
7014 - $all_vector_ids[] = $vector['id'];
7015 - }
7016 - }
7017 - }
7018 - }
7019 - } else {
7020 - // Single entry: the entry_id IS the vector ID
7021 - $all_vector_ids[] = $entry_id;
7022 - }
7023 - } else {
7024 - $wordpress_entries[] = $entry;
7025 - }
7026 - }
7027 -
7028 - // =============================================
7029 - // PHASE 2: Single batch delete to Pinecone
7030 - // =============================================
7031 - if (!empty($all_vector_ids)) {
7032 - $all_vector_ids = array_values(array_unique($all_vector_ids));
7033 - $pinecone_success = true;
7034 - $batches = array_chunk($all_vector_ids, 100);
7035 -
7036 - foreach ($batches as $batch) {
7037 - $delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
7038 - 'headers' => array(
7039 - 'Api-Key' => $api_key,
7040 - 'accept' => 'application/json',
7041 - 'content-type' => 'application/json'
7042 - ),
7043 - 'body' => wp_json_encode(array('ids' => $batch)),
7044 - 'timeout' => 60
7045 - ));
7046 -
7047 - if (is_wp_error($delete_response)) {
7048 - $pinecone_success = false;
7049 - $errors[] = 'Pinecone batch deletion failed: ' . $delete_response->get_error_message();
7050 - MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed: ' . $delete_response->get_error_message());
7051 - } else {
7052 - $response_code = wp_remote_retrieve_response_code($delete_response);
7053 - if ($response_code !== 200) {
7054 - $pinecone_success = false;
7055 - $response_body = wp_remote_retrieve_body($delete_response);
7056 - $errors[] = "Pinecone API error (HTTP $response_code)";
7057 - MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed (HTTP ' . $response_code . ')', array('response' => substr($response_body, 0, 200)));
7058 - }
7059 - }
7060 - }
7061 -
7062 - // Mark all pinecone entries based on batch result
7063 - foreach ($pinecone_entry_ids as $eid) {
7064 - if ($pinecone_success) {
7065 - $success_ids[] = $eid;
7066 - } else {
7067 - $failed_ids[] = $eid;
7068 - }
7069 - }
7070 - }
7071 -
7072 - // =============================================
7073 - // PHASE 3: WordPress database deletions
7074 - // =============================================
7075 - foreach ($wordpress_entries as $entry) {
7076 - $entry_id = sanitize_text_field($entry['id'] ?? '');
7077 - $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
7078 - $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
7079 -
7080 - if (empty($entry_id)) {
7081 - continue;
7082 - }
7083 -
7084 - try {
7085 - if ($is_group && !empty($source_url)) {
7086 - $result = $wpdb->delete(
7087 - $table_name,
7088 - array('source_url' => $source_url),
7089 - array('%s')
7090 - );
7091 - } else {
7092 - wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
7093 - $result = $wpdb->delete(
7094 - $table_name,
7095 - array('id' => intval($entry_id)),
7096 - array('%d')
7097 - );
7098 - }
7099 -
7100 - if ($result !== false) {
7101 - $success_ids[] = $entry_id;
7102 - } else {
7103 - $failed_ids[] = $entry_id;
7104 - $errors[] = "Database error for entry: $entry_id";
7105 - }
7106 - } catch (Exception $e) {
7107 - $failed_ids[] = $entry_id;
7108 - $errors[] = $e->getMessage();
7109 - }
7110 - }
7111 -
7112 - wp_send_json_success(array(
7113 - 'success_ids' => $success_ids,
7114 - 'failed_ids' => $failed_ids,
7115 - 'errors' => $errors,
7116 - 'total_processed' => count($success_ids) + count($failed_ids)
7117 - ));
7118 -
7119 - exit;
7120 -}
7121 -
7122 -/**
7123 - * Get hierarchical roles for dropdown
7124 - */
7125 -public function mxchat_get_role_options() {
7126 - return array(
7127 - 'public' => __('Public (Everyone)', 'mxchat'),
7128 - 'logged_in' => __('Logged In Users', 'mxchat'),
7129 - 'subscriber' => __('Subscribers & Above', 'mxchat'),
7130 - 'contributor' => __('Contributors & Above', 'mxchat'),
7131 - 'author' => __('Authors & Above', 'mxchat'),
7132 - 'editor' => __('Editors & Above', 'mxchat'),
7133 - 'administrator' => __('Administrators Only', 'mxchat')
7134 - );
7135 -}
7136 -
7137 -/**
7138 - * Check if user has access to content based on role restriction
7139 - */
7140 -public function mxchat_user_has_content_access($role_restriction) {
7141 - // Public content is always accessible
7142 - if ($role_restriction === 'public' || empty($role_restriction)) {
7143 - return true;
7144 - }
7145 -
7146 - // Check if user is logged in for logged_in restriction
7147 - if ($role_restriction === 'logged_in') {
7148 - return is_user_logged_in();
7149 - }
7150 -
7151 - // If not logged in, no access to role-restricted content
7152 - if (!is_user_logged_in()) {
7153 - return false;
7154 - }
7155 -
7156 - $user = wp_get_current_user();
7157 - $user_roles = $user->roles;
7158 -
7159 - if (empty($user_roles)) {
7160 - return false;
7161 - }
7162 -
7163 - // Define role hierarchy (higher number = higher access)
7164 - $hierarchy = array(
7165 - 'subscriber' => 1,
7166 - 'contributor' => 2,
7167 - 'author' => 3,
7168 - 'editor' => 4,
7169 - 'administrator' => 5
7170 - );
7171 -
7172 - // Get required level
7173 - $required_level = isset($hierarchy[$role_restriction]) ? $hierarchy[$role_restriction] : 0;
7174 -
7175 - // Check if user has required level or higher
7176 - foreach ($user_roles as $user_role) {
7177 - $user_level = isset($hierarchy[$user_role]) ? $hierarchy[$user_role] : 0;
7178 - if ($user_level >= $required_level) {
7179 - return true;
7180 - }
7181 - }
7182 -
7183 - return false;
7184 -}
7185 -
7186 -/**
7187 - * Handle role restriction updates via AJAX
7188 - * Removed cache clearing call since we removed caching
7189 - */
7190 -public function ajax_mxchat_update_role_restriction() {
7191 - // Verify nonce and permissions
7192 - if (!check_ajax_referer('mxchat_update_role_nonce', 'nonce', false)) {
7193 - wp_send_json_error('Invalid nonce');
7194 - exit;
7195 - }
7196 -
7197 - if (!current_user_can('manage_options')) {
7198 - wp_send_json_error('Unauthorized access');
7199 - exit;
7200 - }
7201 -
7202 - $entry_id = isset($_POST['entry_id']) ? sanitize_text_field($_POST['entry_id']) : '';
7203 - $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
7204 - $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
7205 -
7206 - if (empty($entry_id)) {
7207 - wp_send_json_error('Invalid entry ID');
7208 - exit;
7209 - }
7210 -
7211 - // Get knowledge manager instance to validate role restriction
7212 - $knowledge_manager = MxChat_Knowledge_Manager::get_instance();
7213 - $valid_roles = array_keys($knowledge_manager->mxchat_get_role_options());
7214 - if (!in_array($role_restriction, $valid_roles)) {
7215 - wp_send_json_error('Invalid role restriction');
7216 - exit;
7217 - }
7218 -
7219 - global $wpdb;
7220 -
7221 - if ($data_source === 'pinecone') {
7222 - // Handle Pinecone role restriction (stored separately in WordPress table)
7223 - $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7224 3329
7225 - // Use REPLACE to insert or update the role restriction
7226 - $result = $wpdb->replace(
7227 - $roles_table,
7228 - array(
7229 - 'vector_id' => $entry_id,
7230 - 'role_restriction' => $role_restriction,
7231 - 'updated_at' => current_time('mysql')
7232 - ),
7233 - array('%s', '%s', '%s')
7234 - );
7235 -
7236 - // No cache clearing needed since we removed caching
7237 -
7238 - } else {
7239 - // Handle WordPress database role restriction (existing functionality)
7240 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7241 -
7242 - $result = $wpdb->update(
3330 + $wpdb->delete(
7243 3331 $table_name,
7244 - array('role_restriction' => $role_restriction),
7245 - array('id' => absint($entry_id)),
7246 - array('%s'),
7247 - array('%d')
7248 - );
7249 - }
7250 -
7251 - if ($result === false) {
7252 - wp_send_json_error('Database update failed: ' . $wpdb->last_error);
7253 - exit;
7254 - }
7255 -
7256 - wp_send_json_success(array(
7257 - 'message' => 'Role restriction updated successfully',
7258 - 'role_restriction' => $role_restriction,
7259 - 'data_source' => $data_source,
7260 - 'entry_id' => $entry_id
7261 - ));
7262 - exit;
7263 -}
7264 -
7265 -// ========================================
7266 -// ROLE-BASED CONTENT RESTRICTIONS - NEW FUNCTIONS
7267 -// Add these to your MxChat_Knowledge_Manager class
7268 -// ========================================
7269 -
7270 -/**
7271 - * Initialize role-based content hooks
7272 - * Add this call to your __construct() or mxchat_init_hooks() method
7273 - */
7274 -private function mxchat_init_role_hooks() {
7275 - // AJAX handlers for tag-role mappings
7276 - add_action('wp_ajax_mxchat_add_tag_role_mapping', array($this, 'ajax_add_tag_role_mapping'));
7277 - add_action('wp_ajax_mxchat_delete_tag_role_mapping', array($this, 'ajax_delete_tag_role_mapping'));
7278 - add_action('wp_ajax_mxchat_get_tag_role_mappings', array($this, 'ajax_get_tag_role_mappings'));
7279 - add_action('wp_ajax_mxchat_bulk_update_tag_roles', array($this, 'ajax_bulk_update_tag_roles'));
7280 -
7281 - // Hook to automatically update role restrictions when tags are added/removed
7282 - add_action('set_object_terms', array($this, 'handle_tag_change'), 10, 6);
7283 -
7284 - // Hook to apply role restrictions on auto-sync
7285 - add_action('mxchat_content_stored', array($this, 'apply_role_restriction_after_storage'), 10, 2);
7286 -}
7287 -
7288 -/**
7289 - * Add tag-role mapping via AJAX
7290 - */
7291 -public function ajax_add_tag_role_mapping() {
7292 - // Verify nonce and permissions
7293 - check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7294 -
7295 - if (!current_user_can('manage_options')) {
7296 - wp_send_json_error('Unauthorized access');
7297 - exit;
7298 - }
7299 -
7300 - $tag_input = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
7301 - $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
7302 -
7303 - if (empty($tag_input)) {
7304 - wp_send_json_error('Please enter a tag name or slug');
7305 - exit;
7306 - }
7307 -
7308 - // Validate role restriction
7309 - $valid_roles = array_keys($this->mxchat_get_role_options());
7310 - if (!in_array($role_restriction, $valid_roles)) {
7311 - wp_send_json_error('Invalid role restriction');
7312 - exit;
7313 - }
7314 -
7315 - // Resolve the tag by slug first, then fall back to its display name, so users can
7316 - // enter either "premium-content" or "Premium Content". (plan b8bcf5 — the field is
7317 - // labeled by name but previously validated by slug only, producing the confusing
7318 - // "Tag does not exist in WordPress" error when a real tag's name was typed.)
7319 - $term = get_term_by('slug', $tag_input, 'post_tag');
7320 - if (!$term) {
7321 - $term = get_term_by('name', $tag_input, 'post_tag');
7322 - }
7323 - if (!$term) {
7324 - wp_send_json_error('No tag with that name or slug exists yet. Create it under Posts → Tags first, then enter its name or slug.');
7325 - exit;
7326 - }
7327 -
7328 - // Always key the mapping by the RESOLVED slug — apply_role_restriction_to_post()
7329 - // compares against each post's tag slugs, so the stored key must be a slug,
7330 - // never the raw (possibly display-name) input.
7331 - $tag_slug = $term->slug;
7332 -
7333 - // Get existing mappings
7334 - $mappings = get_option('mxchat_tag_role_mappings', array());
7335 -
7336 - // Check if mapping already exists
7337 - if (isset($mappings[$tag_slug])) {
7338 - wp_send_json_error('Mapping for this tag already exists');
7339 - exit;
7340 - }
7341 -
7342 - // Add new mapping
7343 - $mappings[$tag_slug] = $role_restriction;
7344 - update_option('mxchat_tag_role_mappings', $mappings);
7345 -
7346 - wp_send_json_success(array(
7347 - 'message' => 'Tag-role mapping added successfully',
7348 - 'tag_slug' => $tag_slug,
7349 - 'role_restriction' => $role_restriction
7350 - ));
7351 - exit;
7352 -}
7353 -
7354 -/**
7355 - * Delete tag-role mapping via AJAX
7356 - */
7357 -public function ajax_delete_tag_role_mapping() {
7358 - // Verify nonce and permissions
7359 - check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7360 -
7361 - if (!current_user_can('manage_options')) {
7362 - wp_send_json_error('Unauthorized access');
7363 - exit;
7364 - }
7365 -
7366 - $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
7367 -
7368 - if (empty($tag_slug)) {
7369 - wp_send_json_error('Tag slug is required');
7370 - exit;
7371 - }
7372 -
7373 - // Get existing mappings
7374 - $mappings = get_option('mxchat_tag_role_mappings', array());
7375 -
7376 - // Check if mapping exists
7377 - if (!isset($mappings[$tag_slug])) {
7378 - wp_send_json_error('Mapping does not exist');
7379 - exit;
7380 - }
7381 -
7382 - // Remove mapping
7383 - unset($mappings[$tag_slug]);
7384 - update_option('mxchat_tag_role_mappings', $mappings);
7385 -
7386 - wp_send_json_success(array(
7387 - 'message' => 'Tag-role mapping deleted successfully',
7388 - 'tag_slug' => $tag_slug
7389 - ));
7390 - exit;
7391 -}
7392 -
7393 -/**
7394 - * Get all tag-role mappings via AJAX
7395 - */
7396 -public function ajax_get_tag_role_mappings() {
7397 - // Verify nonce and permissions
7398 - check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7399 -
7400 - if (!current_user_can('manage_options')) {
7401 - wp_send_json_error('Unauthorized access');
7402 - exit;
7403 - }
7404 -
7405 - // Get mappings
7406 - $mappings = get_option('mxchat_tag_role_mappings', array());
7407 - $role_options = $this->mxchat_get_role_options();
7408 -
7409 - $formatted_mappings = array();
7410 -
7411 - foreach ($mappings as $tag_slug => $role_restriction) {
7412 - // Get tag object
7413 - $term = get_term_by('slug', $tag_slug, 'post_tag');
7414 -
7415 - // Count posts with this tag
7416 - $post_count = 0;
7417 - if ($term) {
7418 - $post_count = $term->count;
7419 - }
7420 -
7421 - $formatted_mappings[] = array(
7422 - 'tag_slug' => $tag_slug,
7423 - 'role_restriction' => $role_restriction,
7424 - 'role_label' => $role_options[$role_restriction] ?? $role_restriction,
7425 - 'post_count' => $post_count
7426 - );
7427 - }
7428 -
7429 - wp_send_json_success(array(
7430 - 'mappings' => $formatted_mappings
7431 - ));
7432 - exit;
7433 -}
7434 -
7435 -/**
7436 - * Bulk update role restrictions for all existing content with mapped tags
7437 - */
7438 -public function ajax_bulk_update_tag_roles() {
7439 - // Verify nonce and permissions
7440 - check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7441 -
7442 - if (!current_user_can('manage_options')) {
7443 - wp_send_json_error('Unauthorized access');
7444 - exit;
7445 - }
7446 -
7447 - // Get mappings
7448 - $mappings = get_option('mxchat_tag_role_mappings', array());
7449 -
7450 - if (empty($mappings)) {
7451 - wp_send_json_error('No tag-role mappings found');
7452 - exit;
7453 - }
7454 -
7455 - global $wpdb;
7456 -
7457 - // Check if using Pinecone
7458 - $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
7459 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7460 -
7461 - $updated_count = 0;
7462 - $details = array();
7463 -
7464 - foreach ($mappings as $tag_slug => $role_restriction) {
7465 - // Get all posts with this tag
7466 - $posts = get_posts(array(
7467 - 'tag' => $tag_slug,
7468 - 'post_type' => 'any',
7469 - 'posts_per_page' => -1,
7470 - 'fields' => 'ids',
7471 - 'post_status' => 'publish'
7472 - ));
7473 -
7474 - if (empty($posts)) {
7475 - continue;
7476 - }
7477 -
7478 - $tag_updated = 0;
7479 -
7480 - foreach ($posts as $post_id) {
7481 - $source_url = get_permalink($post_id);
7482 - if (!$source_url) {
7483 - continue;
7484 - }
7485 -
7486 - if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
7487 - // Update Pinecone role restriction
7488 - $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7489 - $vector_id = md5($source_url);
7490 -
7491 - $result = $wpdb->replace(
7492 - $roles_table,
7493 - array(
7494 - 'vector_id' => $vector_id,
7495 - 'role_restriction' => $role_restriction,
7496 - 'updated_at' => current_time('mysql')
7497 - ),
7498 - array('%s', '%s', '%s')
7499 - );
7500 - } else {
7501 - // Update WordPress DB
7502 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7503 -
7504 - $result = $wpdb->update(
7505 - $table_name,
7506 - array('role_restriction' => $role_restriction),
7507 - array('source_url' => $source_url),
7508 - array('%s'),
7509 - array('%s')
7510 - );
7511 - }
7512 -
7513 - if ($result !== false) {
7514 - $tag_updated++;
7515 - $updated_count++;
7516 - }
7517 - }
7518 -
7519 - if ($tag_updated > 0) {
7520 - $details[] = sprintf(
7521 - 'Tag "%s" (%s): %d posts updated',
7522 - $tag_slug,
7523 - $role_restriction,
7524 - $tag_updated
7525 - );
7526 - }
7527 - }
7528 -
7529 - wp_send_json_success(array(
7530 - 'message' => 'Bulk update completed',
7531 - 'updated_count' => $updated_count,
7532 - 'tags_processed' => count($mappings),
7533 - 'details' => $details
7534 - ));
7535 - exit;
7536 -}
7537 -
7538 -/**
7539 - * Handle tag changes on posts (when tags are added or removed)
7540 - */
7541 -public function handle_tag_change($object_id, $terms, $tt_ids, $taxonomy, $append, $old_tt_ids) {
7542 - // Only process post tags
7543 - if ($taxonomy !== 'post_tag') {
7544 - return;
7545 - }
7546 -
7547 - // Get tag-role mappings
7548 - $mappings = get_option('mxchat_tag_role_mappings', array());
7549 -
7550 - if (empty($mappings)) {
7551 - return;
7552 - }
7553 -
7554 - // Get the post's URL
7555 - $source_url = get_permalink($object_id);
7556 - if (!$source_url) {
7557 - return;
7558 - }
7559 -
7560 - // Determine the highest role restriction based on tags
7561 - $highest_role = 'public';
7562 - $role_hierarchy = array(
7563 - 'public' => 0,
7564 - 'logged_in' => 1,
7565 - 'subscriber' => 2,
7566 - 'contributor' => 3,
7567 - 'author' => 4,
7568 - 'editor' => 5,
7569 - 'administrator' => 6
7570 - );
7571 -
7572 - // Get all current tags for the post
7573 - $current_tags = wp_get_post_tags($object_id, array('fields' => 'slugs'));
7574 -
7575 - // Find the highest role restriction among the tags
7576 - foreach ($current_tags as $tag_slug) {
7577 - if (isset($mappings[$tag_slug])) {
7578 - $role = $mappings[$tag_slug];
7579 - if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
7580 - $highest_role = $role;
7581 - }
7582 - }
7583 - }
7584 -
7585 - // Update the role restriction in the database
7586 - global $wpdb;
7587 -
7588 - // Check if using Pinecone
7589 - $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
7590 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7591 -
7592 - if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
7593 - // Update Pinecone role restriction
7594 - $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7595 - $vector_id = md5($source_url);
7596 -
7597 - $wpdb->replace(
7598 - $roles_table,
7599 - array(
7600 - 'vector_id' => $vector_id,
7601 - 'role_restriction' => $highest_role,
7602 - 'updated_at' => current_time('mysql')
7603 - ),
7604 - array('%s', '%s', '%s')
7605 - );
7606 - } else {
7607 - // Update WordPress DB
7608 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7609 -
7610 - $wpdb->update(
7611 - $table_name,
7612 - array('role_restriction' => $highest_role),
7613 3332 array('source_url' => $source_url),
7614 - array('%s'),
7615 3333 array('%s')
7616 3334 );
7617 3335 }
7618 3336 }
7619 -
7620 -/**
7621 - * Apply role restriction after content is stored (for auto-sync)
7622 - */
7623 -public function apply_role_restriction_after_storage($post_id, $source_url) {
7624 - // Get tag-role mappings
7625 - $mappings = get_option('mxchat_tag_role_mappings', array());
7626 3337
7627 - if (empty($mappings)) {
7628 - return;
7629 - }
7630 -
7631 - // Get all tags for the post
7632 - $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
7633 -
7634 - if (empty($post_tags)) {
7635 - return;
7636 - }
7637 -
7638 - // Determine the highest role restriction based on tags
7639 - $highest_role = 'public';
7640 - $role_hierarchy = array(
7641 - 'public' => 0,
7642 - 'logged_in' => 1,
7643 - 'subscriber' => 2,
7644 - 'contributor' => 3,
7645 - 'author' => 4,
7646 - 'editor' => 5,
7647 - 'administrator' => 6
7648 - );
7649 -
7650 - foreach ($post_tags as $tag_slug) {
7651 - if (isset($mappings[$tag_slug])) {
7652 - $role = $mappings[$tag_slug];
7653 - if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
7654 - $highest_role = $role;
7655 - }
7656 - }
7657 - }
7658 -
7659 - // If no restricted tags found, return (leave as public)
7660 - if ($highest_role === 'public') {
7661 - return;
7662 - }
7663 -
7664 - // Update the role restriction
7665 - global $wpdb;
7666 -
7667 - // Check if using Pinecone
7668 - $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
7669 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7670 -
7671 - if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
7672 - // Update Pinecone role restriction
7673 - $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7674 - $vector_id = md5($source_url);
7675 -
7676 - $wpdb->replace(
7677 - $roles_table,
7678 - array(
7679 - 'vector_id' => $vector_id,
7680 - 'role_restriction' => $highest_role,
7681 - 'updated_at' => current_time('mysql')
7682 - ),
7683 - array('%s', '%s', '%s')
7684 - );
7685 - } else {
7686 - // Update WordPress DB
7687 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7688 -
7689 - $wpdb->update(
7690 - $table_name,
7691 - array('role_restriction' => $highest_role),
7692 - array('source_url' => $source_url),
7693 - array('%s'),
7694 - array('%s')
7695 - );
7696 - }
7697 -}
7698 -
7699 -
7700 3338 // ========================================
7701 3339 // HELPER METHODS
7702 3340 // ========================================
7703 3341
@@ -7764,1062 +3402,8 @@
7764 3402 */
7765 3403 private function mxchat_get_pinecone_manager() {
7766 3404 return MxChat_Pinecone_Manager::get_instance();
7767 3405 }
7768 -
7769 -
7770 - // ========================================
7771 -// DATABASE QUEUE TABLE MANAGEMENT
7772 -// ========================================
7773 -
7774 -/**
7775 - * Create queue table on plugin activation
7776 - * Call this from your plugin activation hook
7777 - */
7778 -public function mxchat_create_queue_table() {
7779 - global $wpdb;
7780 -
7781 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7782 - $charset_collate = $wpdb->get_charset_collate();
7783 -
7784 - $sql = "CREATE TABLE IF NOT EXISTS $table_name (
7785 - id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
7786 - queue_id varchar(64) NOT NULL,
7787 - item_type varchar(20) NOT NULL,
7788 - item_data longtext NOT NULL,
7789 - status varchar(20) NOT NULL DEFAULT 'pending',
7790 - bot_id varchar(50) NOT NULL DEFAULT 'default',
7791 - priority int(11) NOT NULL DEFAULT 0,
7792 - attempts int(11) NOT NULL DEFAULT 0,
7793 - max_attempts int(11) NOT NULL DEFAULT 3,
7794 - error_message text DEFAULT NULL,
7795 - created_at datetime NOT NULL,
7796 - started_at datetime DEFAULT NULL,
7797 - completed_at datetime DEFAULT NULL,
7798 - PRIMARY KEY (id),
7799 - KEY queue_id (queue_id),
7800 - KEY status (status),
7801 - KEY item_type (item_type),
7802 - KEY priority (priority)
7803 - ) $charset_collate;";
7804 -
7805 - require_once(ABSPATH . 'wp-admin/includes/upgrade.php');
7806 - dbDelta($sql);
7807 -
7808 - // Also create a meta table for queue metadata
7809 - $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7810 -
7811 - $meta_sql = "CREATE TABLE IF NOT EXISTS $meta_table (
7812 - id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
7813 - queue_id varchar(64) NOT NULL,
7814 - meta_key varchar(255) NOT NULL,
7815 - meta_value longtext,
7816 - PRIMARY KEY (id),
7817 - KEY queue_id (queue_id),
7818 - KEY meta_key (meta_key)
7819 - ) $charset_collate;";
7820 -
7821 - dbDelta($meta_sql);
7822 -}
7823 -
7824 -/**
7825 - * Add items to the processing queue
7826 - *
7827 - * @param string $queue_id Unique identifier for this queue batch
7828 - * @param string $item_type Type of item (url, pdf_page)
7829 - * @param array $items Array of items to queue
7830 - * @param string $bot_id Bot ID for processing
7831 - * @return int Number of items queued
7832 - */
7833 -private function mxchat_add_to_queue($queue_id, $item_type, $items, $bot_id = 'default') {
7834 - global $wpdb;
7835 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7836 -
7837 - $queued_count = 0;
7838 - $priority = 0;
7839 -
7840 - foreach ($items as $item) {
7841 - $result = $wpdb->insert(
7842 - $table_name,
7843 - array(
7844 - 'queue_id' => $queue_id,
7845 - 'item_type' => $item_type,
7846 - 'item_data' => wp_json_encode($item),
7847 - 'status' => 'pending',
7848 - 'bot_id' => $bot_id,
7849 - 'priority' => $priority,
7850 - 'attempts' => 0,
7851 - 'max_attempts' => 3,
7852 - 'created_at' => current_time('mysql')
7853 - ),
7854 - array('%s', '%s', '%s', '%s', '%s', '%d', '%d', '%d', '%s')
7855 - );
7856 -
7857 - if ($result) {
7858 - $queued_count++;
7859 - }
7860 -
7861 - $priority++; // Process in order
7862 - }
7863 -
7864 - return $queued_count;
7865 -}
7866 -
7867 -/**
7868 - * Store queue metadata (total counts, source URL, etc.)
7869 - */
7870 -private function mxchat_set_queue_meta($queue_id, $meta_key, $meta_value) {
7871 - global $wpdb;
7872 - $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7873 -
7874 - // Check if meta exists
7875 - $existing = $wpdb->get_var($wpdb->prepare(
7876 - "SELECT id FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
7877 - $queue_id,
7878 - $meta_key
7879 - ));
7880 -
7881 - if ($existing) {
7882 - // Update
7883 - $wpdb->update(
7884 - $meta_table,
7885 - array('meta_value' => maybe_serialize($meta_value)),
7886 - array('queue_id' => $queue_id, 'meta_key' => $meta_key),
7887 - array('%s'),
7888 - array('%s', '%s')
7889 - );
7890 - } else {
7891 - // Insert
7892 - $wpdb->insert(
7893 - $meta_table,
7894 - array(
7895 - 'queue_id' => $queue_id,
7896 - 'meta_key' => $meta_key,
7897 - 'meta_value' => maybe_serialize($meta_value)
7898 - ),
7899 - array('%s', '%s', '%s')
7900 - );
7901 - }
7902 -}
7903 -
7904 -/**
7905 - * Get queue metadata
7906 - */
7907 -private function mxchat_get_queue_meta($queue_id, $meta_key) {
7908 - global $wpdb;
7909 - $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7910 -
7911 - $value = $wpdb->get_var($wpdb->prepare(
7912 - "SELECT meta_value FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
7913 - $queue_id,
7914 - $meta_key
7915 - ));
7916 -
7917 - return maybe_unserialize($value);
7918 -}
7919 -
7920 -// ========================================
7921 -// AJAX QUEUE PROCESSING HANDLERS
7922 -// ========================================
7923 -
7924 -/**
7925 - * AJAX: Get next item from queue to process
7926 - */
7927 -public function ajax_mxchat_get_next_queue_item() {
7928 - // Verify nonce and permissions
7929 - check_ajax_referer('mxchat_queue_nonce', 'nonce');
7930 -
7931 - if (!current_user_can('manage_options')) {
7932 - wp_send_json_error('Unauthorized access');
7933 - }
7934 -
7935 - $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
7936 -
7937 - if (empty($queue_id)) {
7938 - wp_send_json_error('Missing queue ID');
7939 - }
7940 -
7941 - global $wpdb;
7942 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7943 -
7944 - // Get next pending item with retry logic for failed items
7945 - $next_item = $wpdb->get_row($wpdb->prepare(
7946 - "SELECT * FROM $table_name
7947 - WHERE queue_id = %s
7948 - AND status IN ('pending', 'failed')
7949 - AND attempts < max_attempts
7950 - ORDER BY priority ASC, id ASC
7951 - LIMIT 1",
7952 - $queue_id
7953 - ));
7954 -
7955 - if (!$next_item) {
7956 - // No more items - queue complete
7957 - wp_send_json_success(array(
7958 - 'complete' => true,
7959 - 'message' => 'Queue processing complete'
7960 - ));
7961 - }
7962 -
7963 - // Mark item as processing
7964 - $wpdb->update(
7965 - $table_name,
7966 - array(
7967 - 'status' => 'processing',
7968 - 'started_at' => current_time('mysql'),
7969 - 'attempts' => $next_item->attempts + 1
7970 - ),
7971 - array('id' => $next_item->id),
7972 - array('%s', '%s', '%d'),
7973 - array('%d')
7974 - );
7975 -
7976 - wp_send_json_success(array(
7977 - 'complete' => false,
7978 - 'item' => array(
7979 - 'id' => $next_item->id,
7980 - 'type' => $next_item->item_type,
7981 - 'data' => json_decode($next_item->item_data, true),
7982 - 'bot_id' => $next_item->bot_id,
7983 - 'attempt' => $next_item->attempts + 1
7984 - )
7985 - ));
7986 -}
7987 -
7988 -/**
7989 - * AJAX: Process a single queue item
7990 - */
7991 -public function ajax_mxchat_process_queue_item() {
7992 - // Verify nonce and permissions
7993 - check_ajax_referer('mxchat_queue_nonce', 'nonce');
7994 -
7995 - if (!current_user_can('manage_options')) {
7996 - wp_send_json_error('Unauthorized access');
7997 - }
7998 -
7999 - $item_id = isset($_POST['item_id']) ? absint($_POST['item_id']) : 0;
8000 - $item_type = isset($_POST['item_type']) ? sanitize_text_field($_POST['item_type']) : '';
8001 - $item_data = isset($_POST['item_data']) ? $_POST['item_data'] : array();
8002 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
8003 -
8004 - if (empty($item_id) || empty($item_type)) {
8005 - wp_send_json_error('Missing item data');
8006 - }
8007 -
8008 - global $wpdb;
8009 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8010 -
8011 - // Process based on item type
8012 - try {
8013 - set_time_limit(60); // Give processing 60 seconds
8014 -
8015 - $result = false;
8016 - $error_message = '';
8017 -
8018 - // Read item directly from DB to get queue_id and preserve special chars in item_data
8019 - // (POST round-trip through JS mangles characters like apostrophes in URLs)
8020 - $db_item = $wpdb->get_row($wpdb->prepare(
8021 - "SELECT queue_id, item_data FROM $table_name WHERE id = %d",
8022 - $item_id
8023 - ));
8024 - $item_queue_id = $db_item ? $db_item->queue_id : '';
8025 - if ($db_item && !empty($db_item->item_data)) {
8026 - $db_data = json_decode($db_item->item_data, true);
8027 - if (is_array($db_data)) {
8028 - $item_data = $db_data;
8029 - }
8030 - }
8031 -
8032 - switch ($item_type) {
8033 - case 'url':
8034 - $result = $this->mxchat_process_queue_url($item_data, $bot_id, $item_queue_id);
8035 - break;
8036 -
8037 - case 'pdf_page':
8038 - $result = $this->mxchat_process_queue_pdf_page($item_data, $bot_id);
8039 - break;
8040 -
8041 - default:
8042 - throw new Exception('Unknown item type: ' . $item_type);
8043 - }
8044 -
8045 - if (is_wp_error($result)) {
8046 - $error_code = $result->get_error_code();
8047 - // Content errors (empty page, sanitization) are permanent — retrying won't help
8048 - $permanent_codes = array('empty_page', 'empty_after_sanitization', 'no_api_key', 'page_not_found');
8049 - if (in_array($error_code, $permanent_codes)) {
8050 - // Mark as permanently failed — set attempts = max_attempts so it won't be retried
8051 - $current_item = $wpdb->get_row($wpdb->prepare(
8052 - "SELECT max_attempts FROM $table_name WHERE id = %d", $item_id
8053 - ));
8054 - $wpdb->update(
8055 - $table_name,
8056 - array(
8057 - 'status' => 'failed',
8058 - 'error_message' => $result->get_error_message(),
8059 - 'attempts' => $current_item ? $current_item->max_attempts : 3
8060 - ),
8061 - array('id' => $item_id),
8062 - array('%s', '%s', '%d'),
8063 - array('%d')
8064 - );
8065 - wp_send_json_error(array(
8066 - 'message' => $result->get_error_message(),
8067 - 'permanent_failure' => true,
8068 - 'item_id' => $item_id
8069 - ));
8070 - return;
8071 - }
8072 - throw new Exception($result->get_error_message());
8073 - }
8074 -
8075 - if ($result === false) {
8076 - throw new Exception('Processing returned false - item may be empty or invalid');
8077 - }
8078 -
8079 - // Mark as completed
8080 - $wpdb->update(
8081 - $table_name,
8082 - array(
8083 - 'status' => 'completed',
8084 - 'completed_at' => current_time('mysql'),
8085 - 'error_message' => null
8086 - ),
8087 - array('id' => $item_id),
8088 - array('%s', '%s', '%s'),
8089 - array('%d')
8090 - );
8091 -
8092 - wp_send_json_success(array(
8093 - 'processed' => true,
8094 - 'item_id' => $item_id,
8095 - 'message' => 'Item processed successfully'
8096 - ));
8097 -
8098 - } catch (Exception $e) {
8099 - $error_message = $e->getMessage();
8100 -
8101 - // Get current attempt count
8102 - $item = $wpdb->get_row($wpdb->prepare(
8103 - "SELECT attempts, max_attempts FROM $table_name WHERE id = %d",
8104 - $item_id
8105 - ));
8106 -
8107 - // Check if we've exhausted retries
8108 - if ($item && $item->attempts >= $item->max_attempts) {
8109 - // Permanently failed
8110 - $wpdb->update(
8111 - $table_name,
8112 - array(
8113 - 'status' => 'failed',
8114 - 'error_message' => $error_message
8115 - ),
8116 - array('id' => $item_id),
8117 - array('%s', '%s'),
8118 - array('%d')
8119 - );
8120 -
8121 - wp_send_json_error(array(
8122 - 'message' => 'Item failed after maximum attempts: ' . $error_message,
8123 - 'permanent_failure' => true,
8124 - 'item_id' => $item_id
8125 - ));
8126 - } else {
8127 - // Mark for retry
8128 - $wpdb->update(
8129 - $table_name,
8130 - array(
8131 - 'status' => 'failed',
8132 - 'error_message' => $error_message
8133 - ),
8134 - array('id' => $item_id),
8135 - array('%s', '%s'),
8136 - array('%d')
8137 - );
8138 -
8139 - wp_send_json_error(array(
8140 - 'message' => 'Item processing failed, will retry: ' . $error_message,
8141 - 'can_retry' => true,
8142 - 'item_id' => $item_id,
8143 - 'attempts' => $item ? $item->attempts : 0
8144 - ));
8145 - }
8146 - }
8147 -}
8148 -
8149 -/**
8150 - * Process a URL from the queue
8151 - */
8152 -private function mxchat_process_queue_url($item_data, $bot_id = 'default', $queue_id = '') {
8153 - $url = isset($item_data['url']) ? $item_data['url'] : '';
8154 -
8155 - if (empty($url)) {
8156 - return new WP_Error('invalid_url', 'URL is empty');
8157 - }
8158 -
8159 - // Get bot-specific embedding decision early (needed for both paths) —
8160 - // custom-provider-aware (plan cbd5fd). Error code preserved.
8161 - $bot_options = $this->get_bot_options($bot_id);
8162 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
8163 -
8164 - $preflight = MxChat_Utils::embedding_preflight($options);
8165 - if (!$preflight['ok']) {
8166 - return new WP_Error('no_api_key', $preflight['reason'] . ' (bot: ' . $bot_id . ')');
8167 - }
8168 - $api_key = $preflight['api_key'];
8169 -
8170 - // Check if this is a WooCommerce product URL and WooCommerce is active
8171 - $is_product_url = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
8172 - $content_type = $is_product_url ? 'product' : 'url';
8173 -
8174 - // Try to get WooCommerce product data if it's a product URL
8175 - if ($is_product_url && class_exists('WooCommerce')) {
8176 - $product_content = $this->mxchat_extract_woocommerce_product_content($url);
8177 -
8178 - if (!empty($product_content)) {
8179 - // Successfully extracted WooCommerce product data with pricing
8180 - $result = MxChat_Utils::submit_content_to_db(
8181 - $product_content,
8182 - $url,
8183 - $api_key,
8184 - null,
8185 - $bot_id,
8186 - 'product'
8187 - );
8188 - return $result;
8189 - }
8190 - // If WooCommerce extraction failed, fall through to HTML extraction
8191 - }
8192 -
8193 - // Fetch URL content (fallback for non-products or when WooCommerce extraction fails)
8194 - $is_likely_pdf = (strtolower(pathinfo(parse_url($url, PHP_URL_PATH) ?: '', PATHINFO_EXTENSION)) === 'pdf');
8195 - $response = wp_remote_get($url, array(
8196 - 'timeout' => $is_likely_pdf ? 120 : 30,
8197 - 'redirection' => 5,
8198 - 'user-agent' => mxchat_ingest_user_agent(),
8199 - ));
8200 -
8201 - if (is_wp_error($response)) {
8202 - return $response;
8203 - }
8204 -
8205 - $response_code = wp_remote_retrieve_response_code($response);
8206 - if ($response_code !== 200) {
8207 - return new WP_Error('http_error', 'HTTP ' . $response_code . ' error for: ' . $url);
8208 - }
8209 -
8210 - // Check if URL is a PDF — expand into per-page queue items using the standard PDF pipeline
8211 - if ($this->mxchat_is_pdf_url($url, $response)) {
8212 - return $this->mxchat_expand_pdf_to_queue($url, $response, $bot_id, $queue_id);
8213 - }
8214 -
8215 - $html = wp_remote_retrieve_body($response);
8216 -
8217 - if (empty($html)) {
8218 - return new WP_Error('empty_response', 'Empty response body');
8219 - }
8220 -
8221 - // Extract and sanitize content
8222 - $content = $this->mxchat_extract_main_content($html);
8223 - $sanitized = $this->mxchat_sanitize_content_for_api($content);
8224 -
8225 - if (empty($sanitized)) {
8226 - // Not an error - just no content found (maybe a redirect or empty page)
8227 - return false;
8228 - }
8229 -
8230 - // Submit to database with content_type
8231 - $result = MxChat_Utils::submit_content_to_db(
8232 - $sanitized,
8233 - $url,
8234 - $api_key,
8235 - null,
8236 - $bot_id,
8237 - $content_type
8238 - );
8239 -
8240 - return $result;
8241 -}
8242 -
8243 -/**
8244 - * Expand a PDF URL into per-page queue items using the standard PDF pipeline.
8245 - * Called when a sitemap URL turns out to be a PDF — downloads, parses page count,
8246 - * and adds pdf_page items to the same queue so they process with full progress tracking.
8247 - */
8248 -private function mxchat_expand_pdf_to_queue($pdf_url, $response, $bot_id = 'default', $queue_id = '') {
8249 - set_time_limit(120); // PDFs need extra time for download + parsing
8250 -
8251 - $upload_dir = wp_upload_dir();
8252 - $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
8253 - $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
8254 -
8255 - $response_body = wp_remote_retrieve_body($response);
8256 - if (empty($response_body)) {
8257 - return new WP_Error('empty_pdf', 'Empty PDF response for: ' . $pdf_url);
8258 - }
8259 -
8260 - if (!wp_mkdir_p(dirname($pdf_path))) {
8261 - return new WP_Error('dir_error', 'Failed to create upload directory');
8262 - }
8263 -
8264 - file_put_contents($pdf_path, $response_body);
8265 -
8266 - if (!file_exists($pdf_path)) {
8267 - return new WP_Error('save_error', 'Failed to save PDF file');
8268 - }
8269 -
8270 - try {
8271 - $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
8272 -
8273 - if ($total_pages === false || $total_pages < 1) {
8274 - wp_delete_file($pdf_path);
8275 - return new WP_Error('no_pages', 'PDF has no pages: ' . $pdf_url);
8276 - }
8277 -
8278 - // Build per-page items identical to mxchat_handle_pdf_for_knowledge_base
8279 - $pages = array();
8280 - for ($i = 1; $i <= $total_pages; $i++) {
8281 - $pages[] = array(
8282 - 'pdf_path' => $pdf_path,
8283 - 'pdf_url' => $pdf_url,
8284 - 'page_number' => $i,
8285 - 'total_pages' => $total_pages
8286 - );
8287 - }
8288 -
8289 - // Add pdf_page items to the SAME queue so the JS picks them up automatically
8290 - if (!empty($queue_id)) {
8291 - $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
8292 - } else {
8293 - // Fallback: create a new PDF queue (shouldn't happen in sitemap flow)
8294 - $new_queue_id = 'pdf_' . md5($pdf_url . time());
8295 - $queued_count = $this->mxchat_add_to_queue($new_queue_id, 'pdf_page', $pages, $bot_id);
8296 - $this->mxchat_set_queue_meta($new_queue_id, 'source_url', $pdf_url);
8297 - $this->mxchat_set_queue_meta($new_queue_id, 'queue_type', 'pdf');
8298 - $this->mxchat_set_queue_meta($new_queue_id, 'total_items', $total_pages);
8299 - $this->mxchat_set_queue_meta($new_queue_id, 'bot_id', $bot_id);
8300 - $this->mxchat_set_queue_meta($new_queue_id, 'pdf_path', $pdf_path);
8301 - $this->mxchat_set_queue_meta($new_queue_id, 'created_at', current_time('mysql'));
8302 - }
8303 -
8304 - if ($queued_count === 0) {
8305 - wp_delete_file($pdf_path);
8306 - return new WP_Error('queue_error', 'Failed to add PDF pages to queue');
8307 - }
8308 -
8309 - // Return true so the original URL item is marked complete
8310 - // The new pdf_page items will be processed in subsequent batches
8311 - return true;
8312 -
8313 - } catch (Exception $e) {
8314 - if (file_exists($pdf_path)) {
8315 - wp_delete_file($pdf_path);
8316 - }
8317 - return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
8318 - }
8319 -}
8320 -
8321 -/**
8322 - * Legacy: Process a PDF URL inline during sitemap queue processing.
8323 - * @deprecated Use mxchat_expand_pdf_to_queue instead — kept for reference only.
8324 - */
8325 -private function mxchat_process_pdf_url_inline($pdf_url, $response, $api_key, $bot_id = 'default') {
8326 - set_time_limit(120); // PDFs need more time — downloading + parsing all pages
8327 -
8328 - $upload_dir = wp_upload_dir();
8329 - $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
8330 - $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
8331 -
8332 - $response_body = wp_remote_retrieve_body($response);
8333 - if (empty($response_body)) {
8334 - return new WP_Error('empty_pdf', 'Empty PDF response');
8335 - }
8336 -
8337 - if (!wp_mkdir_p(dirname($pdf_path))) {
8338 - return new WP_Error('dir_error', 'Failed to create upload directory');
8339 - }
8340 -
8341 - file_put_contents($pdf_path, $response_body);
8342 -
8343 - if (!file_exists($pdf_path)) {
8344 - return new WP_Error('save_error', 'Failed to save PDF file');
8345 - }
8346 -
8347 - try {
8348 - mxchat_load_pdf_parser();
8349 - $parser = new \Smalot\PdfParser\Parser();
8350 - $pdf = $parser->parseFile($pdf_path);
8351 - $pages = $pdf->getPages();
8352 - $total_pages = count($pages);
8353 -
8354 - if ($total_pages < 1) {
8355 - wp_delete_file($pdf_path);
8356 - return new WP_Error('no_pages', 'PDF has no pages');
8357 - }
8358 -
8359 - $processed = 0;
8360 - $skipped_pages = array();
8361 -
8362 - for ($i = 0; $i < $total_pages; $i++) {
8363 - $page_num = $i + 1;
8364 - $text = $pages[$i]->getText();
8365 - if (empty($text)) {
8366 - $skipped_pages[] = 'Page ' . $page_num . ': No text could be extracted — page may contain only images, links, or non-standard encoding';
8367 - continue;
8368 - }
8369 -
8370 - $sanitized = $this->mxchat_sanitize_content_for_api($text);
8371 - if (empty($sanitized)) {
8372 - $skipped_pages[] = 'Page ' . $page_num . ': Text was extracted but contained only special characters, control codes, or unsupported content';
8373 - continue;
8374 - }
8375 -
8376 - $metadata = array(
8377 - 'document_type' => 'pdf',
8378 - 'total_pages' => $total_pages,
8379 - 'current_page' => $page_num,
8380 - 'source_url' => $pdf_url,
8381 - );
8382 -
8383 - $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
8384 - $page_url = esc_url($pdf_url . '#page=' . $page_num);
8385 -
8386 - MxChat_Utils::submit_content_to_db(
8387 - $content_with_metadata,
8388 - $page_url,
8389 - $api_key,
8390 - null,
8391 - $bot_id,
8392 - 'pdf'
8393 - );
8394 -
8395 - $processed++;
8396 - }
8397 -
8398 - // Clean up the temp PDF file
8399 - wp_delete_file($pdf_path);
8400 -
8401 - if (!empty($skipped_pages)) {
8402 - error_log('MxChat PDF: Skipped ' . count($skipped_pages) . ' of ' . $total_pages . ' pages: ' . implode('; ', $skipped_pages));
8403 - }
8404 -
8405 - return $processed > 0 ? true : false;
8406 -
8407 - } catch (Exception $e) {
8408 - if (file_exists($pdf_path)) {
8409 - wp_delete_file($pdf_path);
8410 - }
8411 - return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
8412 - }
8413 -}
8414 -
8415 -/**
8416 - * Extract WooCommerce product content including pricing
8417 - *
8418 - * @param string $url The product URL
8419 - * @return string|false Product content with pricing, or false if not found
8420 - */
8421 -private function mxchat_extract_woocommerce_product_content($url) {
8422 - // Try to get product ID from URL
8423 - $product_id = url_to_postid($url);
8424 -
8425 - // If url_to_postid fails, try to extract from URL pattern
8426 - if (!$product_id) {
8427 - $product_slug = '';
8428 -
8429 - // Handle pretty permalinks: /product/product-name/
8430 - if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
8431 - $product_slug = $matches[1];
8432 - }
8433 -
8434 - if (!empty($product_slug)) {
8435 - $product_post = get_page_by_path($product_slug, OBJECT, 'product');
8436 - if ($product_post) {
8437 - $product_id = $product_post->ID;
8438 - }
8439 - }
8440 - }
8441 -
8442 - if (!$product_id) {
8443 - return false;
8444 - }
8445 -
8446 - // Get WooCommerce product object
8447 - $product = wc_get_product($product_id);
8448 -
8449 - if (!$product) {
8450 - return false;
8451 - }
8452 -
8453 - // Build product content with pricing (similar to mxchat_store_product_embedding)
8454 - $title = $product->get_name();
8455 - $description = $product->get_description();
8456 - $short_description = $product->get_short_description();
8457 - $sku = $product->get_sku();
8458 -
8459 - // Get pricing information
8460 - $regular_price = $product->get_regular_price();
8461 - $sale_price = $product->get_sale_price();
8462 - $price = $product->get_price(); // Current active price
8463 -
8464 - // Get currency symbol
8465 - $currency_symbol = get_woocommerce_currency_symbol();
8466 -
8467 - // Format content
8468 - $content = $title . "\n\n";
8469 -
8470 - if (!empty($short_description)) {
8471 - $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
8472 - }
8473 -
8474 - if (!empty($description)) {
8475 - $content .= wp_strip_all_tags($description) . "\n\n";
8476 - }
8477 -
8478 - // Add pricing information
8479 - if (!empty($regular_price)) {
8480 - $content .= "Price: " . $currency_symbol . $regular_price . "\n";
8481 - } elseif (!empty($price)) {
8482 - $content .= "Price: " . $currency_symbol . $price . "\n";
8483 - }
8484 -
8485 - if (!empty($sale_price) && $sale_price !== $regular_price) {
8486 - $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
8487 - }
8488 -
8489 - // Handle variable products - show price range
8490 - if ($product->is_type('variable')) {
8491 - $min_price = $product->get_variation_price('min');
8492 - $max_price = $product->get_variation_price('max');
8493 - if ($min_price !== $max_price) {
8494 - $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
8495 - }
8496 - }
8497 -
8498 - if (!empty($sku)) {
8499 - $content .= "SKU: " . $sku . "\n";
8500 - }
8501 -
8502 - // Get product categories
8503 - $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
8504 - if (!empty($categories) && !is_wp_error($categories)) {
8505 - $content .= "Categories: " . implode(', ', $categories) . "\n";
8506 - }
8507 -
8508 - // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
8509 - $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
8510 - if (!empty($custom_tabs) && is_array($custom_tabs)) {
8511 - foreach ($custom_tabs as $tab) {
8512 - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
8513 - $tab_content = isset($tab['content']) ? $tab['content'] : '';
8514 -
8515 - if (!empty($tab_title) && !empty($tab_content)) {
8516 - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
8517 - }
8518 - }
8519 - }
8520 -
8521 - // Also check for reusable/saved tabs applied to this product
8522 - $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
8523 - if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
8524 - $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
8525 - if (!empty($saved_tabs) && is_array($saved_tabs)) {
8526 - foreach ($applied_saved_tabs as $saved_tab_id) {
8527 - if (isset($saved_tabs[$saved_tab_id])) {
8528 - $tab = $saved_tabs[$saved_tab_id];
8529 - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
8530 - $tab_content = isset($tab['content']) ? $tab['content'] : '';
8531 -
8532 - if (!empty($tab_title) && !empty($tab_content)) {
8533 - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
8534 - }
8535 - }
8536 - }
8537 - }
8538 - }
8539 -
8540 - return $this->mxchat_sanitize_content_for_api($content);
8541 -}
8542 -
8543 -/**
8544 - * Process a PDF page from the queue
8545 - */
8546 -private function mxchat_process_queue_pdf_page($item_data, $bot_id = 'default') {
8547 - $pdf_path = isset($item_data['pdf_path']) ? $item_data['pdf_path'] : '';
8548 - $pdf_url = isset($item_data['pdf_url']) ? $item_data['pdf_url'] : '';
8549 - $page_number = isset($item_data['page_number']) ? absint($item_data['page_number']) : 0;
8550 - $total_pages = isset($item_data['total_pages']) ? absint($item_data['total_pages']) : 0;
8551 -
8552 - if (empty($pdf_path) || !file_exists($pdf_path)) {
8553 - return new WP_Error('pdf_not_found', 'PDF file not found: ' . $pdf_path);
8554 - }
8555 -
8556 - if ($page_number < 1) {
8557 - return new WP_Error('invalid_page', 'Invalid page number');
8558 - }
8559 -
8560 - try {
8561 - mxchat_load_pdf_parser();
8562 - $parser = new \Smalot\PdfParser\Parser();
8563 - $pdf = $parser->parseFile($pdf_path);
8564 - $pages = $pdf->getPages();
8565 -
8566 - if (!isset($pages[$page_number - 1])) {
8567 - return new WP_Error('page_not_found', 'Page ' . $page_number . ' not found in PDF');
8568 - }
8569 -
8570 - $text = $pages[$page_number - 1]->getText();
8571 -
8572 - if (empty($text)) {
8573 - return new WP_Error('empty_page', 'Page ' . $page_number . ': No text could be extracted — page may contain only images, links, or non-standard encoding');
8574 - }
8575 -
8576 - $sanitized = $this->mxchat_sanitize_content_for_api($text);
8577 -
8578 - if (empty($sanitized)) {
8579 - return new WP_Error('empty_after_sanitization', 'Page ' . $page_number . ': Text was extracted but contained only special characters, control codes, or unsupported content that was removed during cleanup');
8580 - }
8581 -
8582 - // Create metadata
8583 - $metadata = array(
8584 - 'document_type' => 'pdf',
8585 - 'total_pages' => $total_pages,
8586 - 'current_page' => $page_number,
8587 - 'source_url' => $pdf_url
8588 - );
8589 -
8590 - $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
8591 - $page_url = esc_url($pdf_url . "#page=" . $page_number);
8592 -
8593 - // Get bot-specific embedding decision — custom-provider-aware
8594 - // (plan cbd5fd). Error code preserved.
8595 - $bot_options = $this->get_bot_options($bot_id);
8596 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
8597 -
8598 - $preflight = MxChat_Utils::embedding_preflight($options);
8599 - if (!$preflight['ok']) {
8600 - return new WP_Error('no_api_key', $preflight['reason'] . ' (bot: ' . $bot_id . ')');
8601 - }
8602 - $api_key = $preflight['api_key'];
8603 -
8604 - // Submit to database - UPDATED 2.5.6: Added content_type 'pdf'
8605 - $result = MxChat_Utils::submit_content_to_db(
8606 - $content_with_metadata,
8607 - $page_url,
8608 - $api_key,
8609 - null,
8610 - $bot_id,
8611 - 'pdf'
8612 - );
8613 -
8614 - return $result;
8615 -
8616 - } catch (Exception $e) {
8617 - return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
8618 - }
8619 -}
8620 -
8621 -/**
8622 - * AJAX: Get queue processing status
8623 - */
8624 -public function ajax_mxchat_get_queue_status() {
8625 - // Verify nonce and permissions
8626 - check_ajax_referer('mxchat_queue_nonce', 'nonce');
8627 -
8628 - if (!current_user_can('manage_options')) {
8629 - wp_send_json_error('Unauthorized access');
8630 - }
8631 -
8632 - $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8633 -
8634 - if (empty($queue_id)) {
8635 - wp_send_json_error('Missing queue ID');
8636 - }
8637 -
8638 - global $wpdb;
8639 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8640 -
8641 - // Get counts by status
8642 - $counts = $wpdb->get_results($wpdb->prepare(
8643 - "SELECT status, COUNT(*) as count
8644 - FROM $table_name
8645 - WHERE queue_id = %s
8646 - GROUP BY status",
8647 - $queue_id
8648 - ), OBJECT_K);
8649 -
8650 - $total = 0;
8651 - $completed = 0;
8652 - $failed = 0;
8653 - $processing = 0;
8654 - $pending = 0;
8655 -
8656 - foreach ($counts as $status => $data) {
8657 - $count = absint($data->count);
8658 - $total += $count;
8659 -
8660 - switch ($status) {
8661 - case 'completed':
8662 - $completed = $count;
8663 - break;
8664 - case 'failed':
8665 - $failed = $count;
8666 - break;
8667 - case 'processing':
8668 - $processing = $count;
8669 - break;
8670 - case 'pending':
8671 - $pending = $count;
8672 - break;
8673 - }
8674 - }
8675 -
8676 - // Calculate percentage
8677 - $percentage = $total > 0 ? round((($completed + $failed) / $total) * 100) : 0;
8678 -
8679 - // Get failed items details (include all failed items, not just those that exhausted retries)
8680 - $failed_items = array();
8681 - if ($failed > 0) {
8682 - $failed_items = $wpdb->get_results($wpdb->prepare(
8683 - "SELECT item_type, item_data, error_message, attempts
8684 - FROM $table_name
8685 - WHERE queue_id = %s
8686 - AND status = 'failed'
8687 - ORDER BY id DESC
8688 - LIMIT 50",
8689 - $queue_id
8690 - ));
8691 - }
8692 -
8693 - // Get queue metadata
8694 - $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
8695 - $queue_type = $this->mxchat_get_queue_meta($queue_id, 'queue_type');
8696 -
8697 - // Determine if queue is complete
8698 - $is_complete = ($pending === 0 && $processing === 0);
8699 -
8700 - wp_send_json_success(array(
8701 - 'queue_id' => $queue_id,
8702 - 'queue_type' => $queue_type,
8703 - 'source_url' => $source_url,
8704 - 'total' => $total,
8705 - 'completed' => $completed,
8706 - 'failed' => $failed,
8707 - 'processing' => $processing,
8708 - 'pending' => $pending,
8709 - 'percentage' => $percentage,
8710 - 'is_complete' => $is_complete,
8711 - 'failed_items' => $failed_items,
8712 - 'status' => $is_complete ? 'complete' : 'processing'
8713 - ));
8714 -}
8715 -
8716 -/**
8717 - * AJAX: Clear completed queue
8718 - */
8719 -public function ajax_mxchat_clear_queue() {
8720 - // Verify nonce and permissions
8721 - check_ajax_referer('mxchat_queue_nonce', 'nonce');
8722 -
8723 - if (!current_user_can('manage_options')) {
8724 - wp_send_json_error('Unauthorized access');
8725 - }
8726 -
8727 - $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8728 -
8729 - if (empty($queue_id)) {
8730 - wp_send_json_error('Missing queue ID');
8731 - }
8732 -
8733 - global $wpdb;
8734 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8735 - $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
8736 -
8737 - // Delete queue items
8738 - $wpdb->delete(
8739 - $table_name,
8740 - array('queue_id' => $queue_id),
8741 - array('%s')
8742 - );
8743 -
8744 - // Delete queue metadata
8745 - $wpdb->delete(
8746 - $meta_table,
8747 - array('queue_id' => $queue_id),
8748 - array('%s')
8749 - );
8750 -
8751 - wp_send_json_success(array(
8752 - 'message' => 'Queue cleared successfully'
8753 - ));
8754 -}
8755 -
8756 -/**
8757 - * AJAX: Retry failed items in queue
8758 - */
8759 -public function ajax_mxchat_retry_failed() {
8760 - // Verify nonce and permissions
8761 - check_ajax_referer('mxchat_queue_nonce', 'nonce');
8762 -
8763 - if (!current_user_can('manage_options')) {
8764 - wp_send_json_error('Unauthorized access');
8765 - }
8766 -
8767 - $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8768 -
8769 - if (empty($queue_id)) {
8770 - wp_send_json_error('Missing queue ID');
8771 - }
8772 -
8773 - global $wpdb;
8774 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8775 -
8776 - // Reset failed items to pending and reset attempt count
8777 - $updated = $wpdb->update(
8778 - $table_name,
8779 - array(
8780 - 'status' => 'pending',
8781 - 'attempts' => 0,
8782 - 'error_message' => null
8783 - ),
8784 - array(
8785 - 'queue_id' => $queue_id,
8786 - 'status' => 'failed'
8787 - ),
8788 - array('%s', '%d', '%s'),
8789 - array('%s', '%s')
8790 - );
8791 -
8792 - wp_send_json_success(array(
8793 - 'message' => 'Reset ' . $updated . ' failed items for retry',
8794 - 'reset_count' => $updated
8795 - ));
8796 -}
8797 -
8798 -
8799 -public function ajax_mxchat_mark_queue_complete() {
8800 - check_ajax_referer('mxchat_queue_nonce', 'nonce');
8801 -
8802 - if (!current_user_can('manage_options')) {
8803 - wp_send_json_error('Unauthorized access');
8804 - }
8805 -
8806 - $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8807 -
8808 - if (empty($queue_id)) {
8809 - wp_send_json_error('Missing queue ID');
8810 - }
8811 -
8812 - // Clear active queue transients
8813 - if (strpos($queue_id, 'sitemap_') === 0) {
8814 - delete_transient('mxchat_active_queue_sitemap');
8815 - } else if (strpos($queue_id, 'pdf_') === 0) {
8816 - delete_transient('mxchat_active_queue_pdf');
8817 - }
8818 -
8819 - wp_send_json_success(array('message' => 'Queue marked as complete'));
8820 -}
8821 -
8822 3406
8823 3407 // ========================================
8824 3408 // STATIC ACCESS METHODS
8825 3409 // ========================================