PluginProbe
MxChat – AI Chatbot & Content Generation for WordPress / 2.2.7
MxChat – AI Chatbot & Content Generation for WordPress v2.2.7
3.2.21 3.2.20 3.2.19 3.2.18 3.2.17 3.2.16 3.2.15 3.2.14 3.2.12 3.2.13 3.2.11 3.2.10 3.2.9 3.2.8 3.2.7 3.2.6 3.2.5 3.2.4 3.2.3 3.2.2 3.2.1 2.0.3 2.0.4 2.0.5 2.0.6 All 152 releases
← All changes | admin/class-knowledge-manager.php +1705 -7062 3.2.18 → 2.2.7 View file →
@@ -9,107 +9,57 @@
9 9 exit; // Exit if accessed directly
10 10 }
11 11
12 12 class MxChat_Knowledge_Manager {
13 -
13 +
14 14 private $options;
15 -
16 - // Post IDs whose vectors were already deleted by mxchat_handle_status_transition this
17 - // request, so the transient-based branch in mxchat_handle_post_update can skip the
18 - // redundant (idempotent but network-visible) second deletion.
19 - private $transition_deleted_posts = array();
20 -
21 - // Post IDs already INDEXED by mxchat_handle_status_transition's arrival edge this
22 - // request. Normal editor publishes fire transition_post_status first, then
23 - // post_updated — without this guard every editor publish would embed twice.
24 - private $transition_indexed_posts = array();
25 -
26 - // Post IDs core has announced an in-flight UPDATE for. pre_post_update fires only
27 - // inside wp_insert_post's update branch and always before wp_transition_post_status,
28 - // so this is an exact "a post_updated is coming later this request" signal — which is
29 - // what makes it safe to arm transition_indexed_posts (plan a664f3).
30 - private $pending_post_update = array();
31 -
15 +
32 16 /**
33 17 * Constructor - Register hooks for content processing
34 18 */
35 -public function __construct() {
36 - $this->options = get_option('mxchat_options', array());
37 - $this->mxchat_init_hooks();
19 + public function __construct() {
20 + $this->options = get_option('mxchat_options', array());
21 + $this->mxchat_init_hooks();
22 + }
38 23
39 - $this->mxchat_init_role_hooks();
40 -}
41 -
42 -/**
43 - * Initialize WordPress hooks for content processing
44 - *
45 - */
46 -private function mxchat_init_hooks() {
47 - // Admin post handlers for form submissions
48 - add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
49 - add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
50 - add_action('admin_post_mxchat_submit_pdf_file', array($this, 'mxchat_handle_pdf_file_submission'));
51 - add_action('admin_post_mxchat_submit_youtube', array($this, 'mxchat_handle_youtube_submission'));
52 - add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
53 -
54 - // AJAX handlers for real-time processing and status updates
55 - add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
56 - add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status'));
57 - add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
58 - add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
59 - add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
60 - add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
61 - add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
62 - add_action('wp_ajax_mxchat_delete_chunks_by_url', array($this, 'ajax_mxchat_delete_chunks_by_url'));
63 - add_action('wp_ajax_mxchat_delete_wordpress_prompt', array($this, 'ajax_mxchat_delete_wordpress_prompt'));
64 - add_action('wp_ajax_mxchat_bulk_delete_knowledge', array($this, 'ajax_mxchat_bulk_delete_knowledge'));
65 - add_action('wp_ajax_mxchat_update_role_restriction', array($this, 'ajax_mxchat_update_role_restriction'));
66 -
67 - // Queue-based processing AJAX handlers
68 - add_action('wp_ajax_mxchat_get_next_queue_item', array($this, 'ajax_mxchat_get_next_queue_item'));
69 - add_action('wp_ajax_mxchat_process_queue_item', array($this, 'ajax_mxchat_process_queue_item'));
70 - add_action('wp_ajax_mxchat_get_queue_status', array($this, 'ajax_mxchat_get_queue_status'));
71 - add_action('wp_ajax_mxchat_clear_queue', array($this, 'ajax_mxchat_clear_queue'));
72 - add_action('wp_ajax_mxchat_retry_failed', array($this, 'ajax_mxchat_retry_failed'));
73 - add_action('wp_ajax_mxchat_get_recent_entries', array($this, 'ajax_mxchat_get_recent_entries'));
74 - add_action('wp_ajax_mxchat_detect_sitemaps', array($this, 'ajax_mxchat_detect_sitemaps'));
75 - add_action('wp_ajax_mxchat_refresh_pinecone_entries', array($this, 'ajax_mxchat_refresh_pinecone_entries'));
76 - add_action('wp_ajax_mxchat_paginate_entries', array($this, 'ajax_mxchat_paginate_entries'));
77 - add_action('wp_ajax_mxchat_get_entry_content', array($this, 'ajax_mxchat_get_entry_content'));
78 - add_action('wp_ajax_mxchat_save_entry_content', array($this, 'ajax_mxchat_save_entry_content'));
79 - add_action('wp_ajax_mxchat_inspect_entry', array($this, 'ajax_mxchat_inspect_entry'));
24 + /**
25 + * Initialize WordPress hooks for content processing
26 + */
27 + private function mxchat_init_hooks() {
28 + // Admin post handlers for form submissions
29 + add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
30 + add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
31 + add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
32 +
33 + // AJAX handlers for real-time processing and status updates
34 + add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
35 + add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status')); // NEW
36 + add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
37 + add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
38 + add_action('wp_ajax_mxchat_manual_batch_process', array($this, 'ajax_manual_batch_process'));
39 + add_action('mxchat_delete_content', array($this, 'mxchat_delete_from_pinecone_by_url'), 10, 1);
80 40
81 - // WordPress post management hooks
82 - add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
83 - add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
84 - add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
85 - add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
86 - // Authoritative unpublish detection: core hands this hook the REAL previous status, so
87 - // removal no longer depends on the mxchat_prev_status_* transients (evictable by persistent
88 - // object caches, never written by paths that bypass wp_update_post, e.g. plugins flipping
89 - // post_status directly and calling wp_transition_post_status themselves).
90 - add_action('transition_post_status', array($this, 'mxchat_handle_status_transition'), 10, 3);
91 -
92 - // ACF hook - fires AFTER ACF fields are saved, ensuring ACF data is available
93 - // Priority 20 to run after ACF's own save (which runs at priority 10)
94 - add_action('acf/save_post', array($this, 'mxchat_handle_acf_save'), 20);
95 -
96 - // One-time cleanup for vectors orphaned by unpublishes that predate the
97 - // transition_post_status handler (plan 816fb1): wp mxchat prune-unpublished
98 - if (defined('WP_CLI') && WP_CLI) {
99 - WP_CLI::add_command('mxchat prune-unpublished', array($this, 'cli_prune_unpublished'));
41 +
42 + // Cron handlers for background processing
43 + add_action('mxchat_process_sitemap_urls', array($this, 'mxchat_process_sitemap_urls_cron'), 10, 5);
44 + add_action('mxchat_process_pdf_pages', array($this, 'mxchat_process_pdf_pages_cron'), 10, 5);
45 +
46 + // WordPress post management hooks
47 + add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
48 + add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
49 + add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
50 + add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
51 + add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
52 + add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
53 +
54 + // WooCommerce product hooks (if WooCommerce is active)
55 + if (class_exists('WooCommerce')) {
56 + add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
57 + add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
58 + add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
59 + }
60 +
100 61 }
101 -
102 - add_action('wp_ajax_mxchat_mark_queue_complete', array($this, 'ajax_mxchat_mark_queue_complete'));
103 -
104 - // WooCommerce product hooks (if WooCommerce is active)
105 - if (class_exists('WooCommerce')) {
106 - add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
107 - add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
108 - add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
109 - add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
110 - }
111 -}
112 62
113 63 /**
114 64 * Get current options (refreshed)
115 65 */
@@ -119,9 +69,315 @@
119 69 }
120 70 return $this->options;
121 71 }
122 72
73 +
74 + /**
75 + * Handle manual batch processing via AJAX
76 + */
77 +public function ajax_manual_batch_process() {
78 + try {
79 + // Verify nonce and permissions
80 + check_ajax_referer('mxchat_status_nonce', 'nonce');
81 +
82 + if (!current_user_can('manage_options')) {
83 + wp_send_json_error('Unauthorized access');
84 + }
85 +
86 + $process_type = sanitize_text_field($_POST['process_type'] ?? '');
87 + $url = sanitize_text_field($_POST['url'] ?? '');
88 +
89 + if (empty($process_type) || empty($url)) {
90 + wp_send_json_error('Missing required parameters');
91 + }
92 +
93 + $processed = 0;
94 +
95 + if ($process_type === 'pdf') {
96 + $processed = $this->mxchat_manual_process_pdf_batch($url);
97 + } elseif ($process_type === 'sitemap') {
98 + $processed = $this->mxchat_manual_process_sitemap_batch($url);
99 + }
100 +
101 + if ($processed > 0) {
102 + wp_send_json_success(array(
103 + 'message' => "Processed {$processed} items successfully",
104 + 'processed' => $processed
105 + ));
106 + } else {
107 + wp_send_json_error('No items were processed');
108 + }
109 +
110 + } catch (Exception $e) {
111 + //error_log('Manual batch process error: ' . $e->getMessage());
112 + wp_send_json_error('Processing failed: ' . $e->getMessage());
113 + }
114 +}
123 115
116 +/**
117 + * Process a small PDF batch manually - DIRECT PROCESSING
118 + */
119 +private function mxchat_manual_process_pdf_batch($pdf_url) {
120 + try {
121 + $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
122 + $status = get_transient($status_key);
123 +
124 + if (!$status || $status['status'] !== 'processing') {
125 + //error_log('Manual PDF: No processing status found');
126 + return 0;
127 + }
128 +
129 + //error_log('Manual PDF: Starting direct processing for ' . $pdf_url);
130 +
131 + // Get current progress
132 + $current_page = $status['processed_pages'] ?? 0;
133 + $total_pages = $status['total_pages'] ?? 0;
134 +
135 + if ($current_page >= $total_pages) {
136 + //error_log('Manual PDF: Already completed');
137 + return 0;
138 + }
139 +
140 + // Try to download the PDF again for processing
141 + $response = wp_remote_get($pdf_url, array('timeout' => 30));
142 +
143 + if (is_wp_error($response)) {
144 + //error_log('Manual PDF: Failed to download PDF: ' . $response->get_error_message());
145 + return 0;
146 + }
147 +
148 + $pdf_content = wp_remote_retrieve_body($response);
149 + if (empty($pdf_content)) {
150 + //error_log('Manual PDF: Empty PDF content');
151 + return 0;
152 + }
153 +
154 + // Save PDF temporarily
155 + $upload_dir = wp_upload_dir();
156 + $temp_pdf_path = trailingslashit($upload_dir['path']) . 'temp_manual_' . time() . '.pdf';
157 + file_put_contents($temp_pdf_path, $pdf_content);
158 +
159 + // Process 2 pages directly
160 + $processed = $this->mxchat_process_pdf_pages_direct($temp_pdf_path, $pdf_url, $current_page, 5);
161 +
162 + // Clean up temp file
163 + if (file_exists($temp_pdf_path)) {
164 + wp_delete_file($temp_pdf_path);
165 + }
166 +
167 + //error_log('Manual PDF: Processed ' . $processed . ' pages');
168 + return $processed;
169 +
170 + } catch (Exception $e) {
171 + //error_log('Manual PDF batch error: ' . $e->getMessage());
172 + return 0;
173 + }
174 +}
175 +
176 +/**
177 + * Process PDF pages directly without cron
178 + */
179 +private function mxchat_process_pdf_pages_direct($pdf_path, $pdf_url, $start_page, $batch_size) {
180 + try {
181 + if (!file_exists($pdf_path)) {
182 + //error_log('Direct PDF: File not found at ' . $pdf_path);
183 + return 0;
184 + }
185 +
186 + $parser = new \Smalot\PdfParser\Parser();
187 + $pdf = $parser->parseFile($pdf_path);
188 + $pages = $pdf->getPages();
189 +
190 + $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
191 + $status = get_transient($status_key);
192 +
193 + if (!$status) {
194 + return 0;
195 + }
196 +
197 + $options = get_option('mxchat_options');
198 + $api_key = $options['api_key'] ?? '';
199 +
200 + if (empty($api_key)) {
201 + //error_log('Direct PDF: No API key');
202 + return 0;
203 + }
204 +
205 + $processed = 0;
206 + $end_page = min($start_page + $batch_size, count($pages));
207 +
208 + for ($i = $start_page; $i < $end_page; $i++) {
209 + try {
210 + $page_number = $i + 1;
211 + $text = $pages[$i]->getText();
212 +
213 + if (empty($text)) {
214 + //error_log('Direct PDF: Empty text on page ' . $page_number);
215 + continue;
216 + }
217 +
218 + $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
219 + if (empty($sanitized_content)) {
220 + //error_log('Direct PDF: No content after sanitization on page ' . $page_number);
221 + continue;
222 + }
223 +
224 + // Generate embedding
225 + $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
226 + if (is_string($embedding_vector)) {
227 + //error_log('Direct PDF: Embedding failed on page ' . $page_number . ': ' . $embedding_vector);
228 + continue;
229 + }
230 +
231 + // Create metadata
232 + $metadata = array(
233 + 'document_type' => 'pdf',
234 + 'total_pages' => count($pages),
235 + 'current_page' => $page_number,
236 + 'source_url' => $pdf_url
237 + );
238 +
239 + $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
240 + $page_url = esc_url($pdf_url . "#page=" . $page_number);
241 +
242 + // Store in database
243 + $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $api_key);
244 +
245 + if (is_wp_error($db_result)) {
246 + //error_log('Direct PDF: DB error on page ' . $page_number . ': ' . $db_result->get_error_message());
247 + continue;
248 + }
249 +
250 + $processed++;
251 + //error_log('Direct PDF: Successfully processed page ' . $page_number);
252 +
253 + // Update status
254 + $status['processed_pages'] = $i + 1;
255 + $status['last_update'] = time();
256 + $status['percentage'] = round(($status['processed_pages'] / $status['total_pages']) * 100);
257 + set_transient($status_key, $status, DAY_IN_SECONDS);
258 +
259 + } catch (Exception $e) {
260 + //error_log('Direct PDF: Error processing page ' . ($i + 1) . ': ' . $e->getMessage());
261 + continue;
262 + }
263 + }
264 +
265 + // Check if completed
266 + if ($status['processed_pages'] >= $status['total_pages']) {
267 + $status['status'] = 'complete';
268 + set_transient($status_key, $status, DAY_IN_SECONDS);
269 + //error_log('Direct PDF: Processing completed');
270 + }
271 +
272 + return $processed;
273 +
274 + } catch (Exception $e) {
275 + //error_log('Direct PDF processing error: ' . $e->getMessage());
276 + return 0;
277 + }
278 +}
279 +
280 +/**
281 + * Process a small sitemap batch manually - DIRECT PROCESSING
282 + */
283 +private function mxchat_manual_process_sitemap_batch($sitemap_url) {
284 + try {
285 + $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
286 + $status = get_transient($status_key);
287 +
288 + if (!$status || $status['status'] !== 'processing') {
289 + return 0;
290 + }
291 +
292 + //error_log('Manual Sitemap: Starting direct processing for ' . $sitemap_url);
293 +
294 + // Re-fetch the sitemap to get URLs
295 + $response = wp_remote_get($sitemap_url, array('timeout' => 30));
296 + if (is_wp_error($response)) {
297 + //error_log('Manual Sitemap: Failed to fetch sitemap');
298 + return 0;
299 + }
300 +
301 + $sitemap_content = wp_remote_retrieve_body($response);
302 + $xml = simplexml_load_string($sitemap_content);
303 +
304 + if (!$xml) {
305 + //error_log('Manual Sitemap: Invalid XML');
306 + return 0;
307 + }
308 +
309 + $urls = array();
310 + foreach ($xml->url as $url_element) {
311 + $urls[] = (string)$url_element->loc;
312 + }
313 +
314 + $current_processed = $status['processed_urls'] ?? 0;
315 + $batch_size = 5;
316 + $processed = 0;
317 +
318 + // Process next 2 URLs
319 + for ($i = $current_processed; $i < min($current_processed + $batch_size, count($urls)); $i++) {
320 + $url = $urls[$i];
321 +
322 + if ($this->mxchat_process_single_url_direct($url)) {
323 + $processed++;
324 + }
325 +
326 + // Update status
327 + $status['processed_urls'] = $i + 1;
328 + $status['last_update'] = time();
329 + $status['percentage'] = round(($status['processed_urls'] / $status['total_urls']) * 100);
330 + set_transient($status_key, $status, DAY_IN_SECONDS);
331 + }
332 +
333 + // Check if completed
334 + if ($status['processed_urls'] >= $status['total_urls']) {
335 + $status['status'] = 'complete';
336 + set_transient($status_key, $status, DAY_IN_SECONDS);
337 + }
338 +
339 + //error_log('Manual Sitemap: Processed ' . $processed . ' URLs');
340 + return $processed;
341 +
342 + } catch (Exception $e) {
343 + //error_log('Manual sitemap batch error: ' . $e->getMessage());
344 + return 0;
345 + }
346 +}
347 +
348 +/**
349 + * Process a single URL directly
350 + */
351 +private function mxchat_process_single_url_direct($url) {
352 + try {
353 + $response = wp_remote_get($url, array('timeout' => 30));
354 + if (is_wp_error($response)) {
355 + return false;
356 + }
357 +
358 + $html = wp_remote_retrieve_body($response);
359 + $content = $this->mxchat_extract_main_content($html);
360 + $sanitized = $this->mxchat_sanitize_content_for_api($content);
361 +
362 + if (empty($sanitized)) {
363 + return false;
364 + }
365 +
366 + $options = get_option('mxchat_options');
367 + $api_key = $options['api_key'] ?? '';
368 +
369 + $result = MxChat_Utils::submit_content_to_db($sanitized, $url, $api_key);
370 +
371 + return !is_wp_error($result);
372 +
373 + } catch (Exception $e) {
374 + //error_log('Single URL processing error: ' . $e->getMessage());
375 + return false;
376 + }
377 +}
378 +
379 +
124 380 // ========================================
125 381 // MAIN CONTENT SUBMISSION HANDLERS
126 382 // ========================================
127 383
@@ -137,30 +393,34 @@
137 393 wp_die(esc_html__('Nonce verification failed.', 'mxchat'));
138 394 }
139 395
140 396 // Sanitize the inputs.
141 - // Use wp_kses_post to allow safe HTML and wp_unslash to remove WordPress added slashes
142 - $article_content = wp_kses_post(wp_unslash($_POST['article_content']));
397 + $article_content = sanitize_textarea_field($_POST['article_content']);
143 398 $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
144 399
145 - // Get bot_id from form submission
146 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
400 + // Get API key for submission
401 + $options = get_option('mxchat_options');
402 + $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
147 403
148 - // Get bot-specific options and API key
149 - $bot_options = $this->get_bot_options($bot_id);
150 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
151 -
152 - // Custom-provider-aware decision; keyless custom sites must pass (plan cbd5fd).
153 - $preflight = MxChat_Utils::embedding_preflight($options);
154 - if (!$preflight['ok']) {
155 - set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
404 + if (strpos($selected_model, 'voyage') === 0) {
405 + $api_key = $options['voyage_api_key'] ?? '';
406 + } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
407 + $api_key = $options['gemini_api_key'] ?? '';
408 + } else {
409 + $api_key = $options['api_key'] ?? '';
410 + }
411 +
412 + if (empty($api_key)) {
413 + set_transient('mxchat_admin_notice_error',
414 + esc_html__('API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
415 + 30
416 + );
156 417 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
157 418 exit;
158 419 }
159 - $api_key = $preflight['api_key'];
160 420
161 - // Use centralized utility function with bot_id
162 - $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key, null, $bot_id);
421 + // Use centralized utility function for storage
422 + $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key);
163 423
164 424 if (is_wp_error($result)) {
165 425 set_transient('mxchat_admin_notice_error',
166 426 esc_html__('Error storing content: ', 'mxchat') . $result->get_error_message(),
@@ -175,253 +435,17 @@
175 435
176 436 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
177 437 exit;
178 438 }
179 -
180 -/**
181 - * Handle the "YouTube" KB import source (admin-post form submission).
182 - *
183 - * Per-video description mode:
184 - * - auto: fetch oEmbed metadata (reliable) + best-effort captions transcript.
185 - * If no usable transcript, index the metadata anyway, tell the admin,
186 - * and bounce back with the manual box pre-filled (never fail silently).
187 - * - manual: the admin's own description is what gets indexed; metadata rides along.
188 - *
189 - * The row is stored with content_type 'youtube' and source_url = the canonical
190 - * watch URL, so re-importing the same video UPDATES the entry (source_url
191 - * duplicate handling in MxChat_Utils::store_in_wordpress_db) — that is also the
192 - * "augment a metadata-only entry" path.
193 - */
194 -public function mxchat_handle_youtube_submission() {
195 - if (!isset($_POST['submit_youtube']) || !current_user_can('manage_options')) {
196 - wp_die(esc_html__('Unauthorized access', 'mxchat'));
197 - }
198 -
199 - check_admin_referer('mxchat_submit_youtube_action', 'mxchat_submit_youtube_nonce');
200 -
201 - $redirect_url = admin_url('admin.php?page=mxchat-prompts');
202 -
203 - $youtube_url = isset($_POST['youtube_url']) ? esc_url_raw(wp_unslash($_POST['youtube_url'])) : '';
204 - $video_id = MxChat_Utils::parse_youtube_id($youtube_url);
205 -
206 - if (empty($video_id)) {
207 - set_transient('mxchat_admin_notice_error',
208 - esc_html__('That does not look like a link to a single YouTube video. Please paste a watch, youtu.be, or Shorts URL.', 'mxchat'),
209 - 30
210 - );
211 - wp_safe_redirect(esc_url($redirect_url));
212 - exit;
213 - }
214 -
215 - $canonical_url = 'https://www.youtube.com/watch?v=' . $video_id;
216 -
217 - $description_mode = (isset($_POST['youtube_description_mode']) && $_POST['youtube_description_mode'] === 'manual') ? 'manual' : 'auto';
218 - $manual_description = isset($_POST['youtube_description']) ? trim(wp_kses_post(wp_unslash($_POST['youtube_description']))) : '';
219 -
220 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
221 -
222 - // Resolve the embedding decision exactly like the sibling handlers —
223 - // custom-provider-aware (plan cbd5fd).
224 - $bot_options = $this->get_bot_options($bot_id);
225 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
226 -
227 - $preflight = MxChat_Utils::embedding_preflight($options);
228 - if (!$preflight['ok']) {
229 - set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
230 - wp_safe_redirect(esc_url($redirect_url));
231 - exit;
232 - }
233 - $api_key = $preflight['api_key'];
234 -
235 - // Metadata is fetched in BOTH modes — it is the reliable half of auto, and in
236 - // manual mode it enriches the indexed text with the real title/channel.
237 - $meta = $this->mxchat_fetch_youtube_oembed($video_id);
238 - $video_title = isset($meta['title']) ? sanitize_text_field($meta['title']) : '';
239 - $video_channel = isset($meta['author_name']) ? sanitize_text_field($meta['author_name']) : '';
240 -
241 - $header_lines = 'YouTube Video: ' . ($video_title !== '' ? $video_title : $canonical_url) . "\n";
242 - if ($video_channel !== '') {
243 - $header_lines .= 'Channel: ' . $video_channel . "\n";
244 - }
245 - $header_lines .= 'URL: ' . $canonical_url . "\n\n";
246 -
247 - $transcript_missing = false;
248 -
249 - if ($description_mode === 'manual') {
250 - if ($manual_description === '') {
251 - set_transient('mxchat_admin_notice_error',
252 - esc_html__('Please write a description for the video, or switch to auto-fetch.', 'mxchat'),
253 - 30
254 - );
255 - wp_safe_redirect(esc_url($redirect_url));
256 - exit;
257 - }
258 - $indexed_text = $header_lines . $manual_description;
259 - } else {
260 - $transcript = $this->mxchat_fetch_youtube_transcript($video_id);
261 -
262 - if (strlen($transcript) >= 200) {
263 - $indexed_text = $header_lines . $transcript;
264 - } else {
265 - // Graceful fallback: captions disabled / blocked / no speech. Auto
266 - // reliably gets metadata; it does NOT guarantee a transcript.
267 - $transcript_missing = true;
268 -
269 - if ($video_title === '' && $video_channel === '') {
270 - // Both halves failed — nothing meaningful to index.
271 - set_transient('mxchat_admin_notice_error',
272 - esc_html__('Could not retrieve any information for that video (no metadata and no captions). Please check the URL, or use the manual description option.', 'mxchat'),
273 - 30
274 - );
275 - wp_safe_redirect(esc_url($redirect_url));
276 - exit;
277 - }
278 -
279 - $indexed_text = $header_lines . sprintf(
280 - /* translators: 1: video title, 2: channel name */
281 - __('A YouTube video titled "%1$s" from the channel %2$s.', 'mxchat'),
282 - $video_title !== '' ? $video_title : $canonical_url,
283 - $video_channel !== '' ? $video_channel : 'YouTube'
284 - );
285 - }
286 - }
287 -
288 - $result = MxChat_Utils::submit_content_to_db($indexed_text, $canonical_url, $api_key, null, $bot_id, 'youtube');
289 -
290 - if (is_wp_error($result)) {
291 - set_transient('mxchat_admin_notice_error',
292 - esc_html__('Error storing video in the knowledge base: ', 'mxchat') . $result->get_error_message(),
293 - 30
294 - );
295 - wp_safe_redirect(esc_url($redirect_url));
296 - exit;
297 - }
298 -
299 - if ($transcript_missing) {
300 - set_transient('mxchat_admin_notice_success',
301 - esc_html__('Video indexed from its title and channel — no captions were available for a transcript. The form below is pre-filled: write your own description and import again to improve matching (it updates the same entry).', 'mxchat'),
302 - 30
303 - );
304 - // Bounce back with prefill args so the page reopens the YouTube form in
305 - // manual mode with the URL + fetched title ready to augment.
306 - $redirect_url = add_query_arg(array(
307 - 'mxchat_yt_prefill' => '1',
308 - 'yt_url' => rawurlencode($canonical_url),
309 - 'yt_title' => rawurlencode($video_title),
310 - ), $redirect_url);
311 - } else {
312 - set_transient('mxchat_admin_notice_success',
313 - esc_html__('YouTube video successfully added to the knowledge base!', 'mxchat'),
314 - 30
315 - );
316 - }
317 -
318 - wp_safe_redirect(esc_url_raw($redirect_url));
319 - exit;
320 -}
321 -
322 -/**
323 - * Fetch YouTube oEmbed metadata for a video (no API key required).
324 - * Returns the decoded array (title, author_name, thumbnail_url, ...) or array().
325 - */
326 -private function mxchat_fetch_youtube_oembed($video_id) {
327 - $oembed_url = 'https://www.youtube.com/oembed?url=' . rawurlencode('https://www.youtube.com/watch?v=' . $video_id) . '&format=json';
328 - $response = wp_remote_get($oembed_url, array('timeout' => 15));
329 - if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
330 - return array();
331 - }
332 - $data = json_decode(wp_remote_retrieve_body($response), true);
333 - return is_array($data) ? $data : array();
334 -}
335 -
336 -/**
337 - * Best-effort captions transcript for a video. Deliberately ISOLATED: this uses
338 - * YouTube's unofficial timedtext route (the caption track list embedded in the
339 - * watch page), which YouTube has broken before and will break again. Every
340 - * failure mode returns '' so a break degrades to the metadata-only import path
341 - * instead of erroring the whole submission. Do not let anything in here throw.
342 - */
343 -private function mxchat_fetch_youtube_transcript($video_id) {
344 - $watch_url = 'https://www.youtube.com/watch?v=' . $video_id . '&hl=en';
345 -
346 - // First try the honest ingest UA; some responses omit the player config for
347 - // bot UAs, so retry once with a browser UA before giving up.
348 - $user_agents = array(
349 - mxchat_ingest_user_agent(),
350 - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
351 - );
352 -
353 - $tracks = array();
354 - foreach ($user_agents as $ua) {
355 - $response = wp_remote_get($watch_url, array(
356 - 'timeout' => 20,
357 - 'user-agent' => $ua,
358 - ));
359 - if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
360 - continue;
361 - }
362 - $body = wp_remote_retrieve_body($response);
363 - if (!is_string($body) || $body === '' || !preg_match('/"captionTracks":(\[.*?\])(?=,")/s', $body, $m)) {
364 - continue;
365 - }
366 - $decoded = json_decode($m[1], true);
367 - if (is_array($decoded) && !empty($decoded)) {
368 - $tracks = $decoded;
369 - break;
370 - }
371 - }
372 -
373 - if (empty($tracks)) {
374 - return '';
375 - }
376 -
377 - // Prefer an English track, else take the first offered.
378 - $chosen = null;
379 - foreach ($tracks as $track) {
380 - if (isset($track['languageCode']) && strpos($track['languageCode'], 'en') === 0) {
381 - $chosen = $track;
382 - break;
383 - }
384 - }
385 - if ($chosen === null) {
386 - $chosen = $tracks[0];
387 - }
388 - if (empty($chosen['baseUrl']) || !is_string($chosen['baseUrl'])) {
389 - return '';
390 - }
391 -
392 - $timedtext = wp_remote_get($chosen['baseUrl'], array('timeout' => 20));
393 - if (is_wp_error($timedtext) || wp_remote_retrieve_response_code($timedtext) !== 200) {
394 - return '';
395 - }
396 - $xml = wp_remote_retrieve_body($timedtext);
397 - if (!is_string($xml) || strpos($xml, '<text') === false) {
398 - return '';
399 - }
400 -
401 - // <text start=".." dur="..">caption</text> — strip tags, decode the
402 - // double-encoded entities timedtext ships, collapse whitespace.
403 - $text = preg_replace('/<[^>]+>/', ' ', $xml);
404 - $text = html_entity_decode(html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8'), ENT_QUOTES | ENT_HTML5, 'UTF-8');
405 - $text = trim(preg_replace('/\s+/u', ' ', $text));
406 -
407 - return $text;
408 -}
409 -
410 439 public function mxchat_is_pdf_url($url, $response) {
411 440 $content_type = wp_remote_retrieve_header($response, 'content-type');
412 441 $file_extension = strtolower(pathinfo($url, PATHINFO_EXTENSION));
413 442
414 - // Check Content-Disposition header for .pdf filename (Google Drive sends this)
415 - $disposition = wp_remote_retrieve_header($response, 'content-disposition');
416 - $has_pdf_disposition = ! empty($disposition) && stripos($disposition, '.pdf') !== false;
417 -
418 - return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf' || $has_pdf_disposition;
443 + return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf';
419 444 }
420 -
421 -
422 -public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response, $bot_id = 'default') {
445 +public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response) {
423 446 if (!current_user_can('manage_options')) {
447 + //error_log(esc_html__('Unauthorized PDF processing attempt', 'mxchat'));
424 448 return false;
425 449 }
426 450
427 451 $pdf_url = esc_url_raw($pdf_url);
@@ -427,8 +451,9 @@
427 451 $pdf_url = esc_url_raw($pdf_url);
428 452 $upload_dir = wp_upload_dir();
429 453
430 454 if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
455 + //error_log(sprintf(esc_html__('Upload directory error: %s', 'mxchat'), esc_html($upload_dir['error'])));
431 456 return false;
432 457 }
433 458
434 459 $pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
@@ -435,12 +460,14 @@
435 460 $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
436 461
437 462 $response_body = wp_remote_retrieve_body($response);
438 463 if (empty($response_body)) {
464 + //error_log(esc_html__('Empty PDF response body', 'mxchat'));
439 465 return false;
440 466 }
441 467
442 468 if (!wp_mkdir_p(dirname($pdf_path))) {
469 + //error_log(sprintf(esc_html__('Failed to create directory for PDF: %s', 'mxchat'), esc_html($pdf_path)));
443 470 return false;
444 471 }
445 472
446 473 try {
@@ -449,915 +476,303 @@
449 476 if (!file_exists($pdf_path)) {
450 477 throw new Exception(__('Failed to save PDF file', 'mxchat'));
451 478 }
452 479
453 - $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
454 -
455 - if ($total_pages === false || $total_pages < 1) {
456 - throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
457 - }
480 + $parser = new \Smalot\PdfParser\Parser();
481 + $pdf = $parser->parseFile($pdf_path);
482 + $total_pages = absint(count($pdf->getPages()));
458 483
459 - // Create unique queue ID
460 - $queue_id = 'pdf_' . md5($pdf_url . time());
461 -
462 - // Create array of pages to process
463 - $pages = array();
464 - for ($i = 1; $i <= $total_pages; $i++) {
465 - $pages[] = array(
466 - 'pdf_path' => $pdf_path,
467 - 'pdf_url' => $pdf_url,
468 - 'page_number' => $i,
469 - 'total_pages' => $total_pages
470 - );
484 + if ($total_pages < 1) {
485 + throw new Exception(__('Invalid PDF: no pages found', 'mxchat'));
471 486 }
472 487
473 - // Add pages to queue
474 - $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
488 + wp_schedule_single_event(time(), 'mxchat_process_pdf_pages', array(
489 + 'pdf_path' => $pdf_path,
490 + 'pdf_url' => $pdf_url,
491 + 'total_pages' => $total_pages,
492 + 'batch_size' => absint(15),
493 + 'batch_pause' => absint(10)
494 + ));
475 495
476 - if ($queued_count === 0) {
477 - wp_delete_file($pdf_path);
478 - throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
479 - }
496 + $status_data = array(
497 + 'total_pages' => $total_pages,
498 + 'processed_pages' => 0,
499 + 'status' => 'processing',
500 + 'last_update' => time()
501 + );
480 502
481 - // Store queue metadata
482 - $this->mxchat_set_queue_meta($queue_id, 'source_url', $pdf_url);
483 - $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
484 - $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
485 - $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
486 - $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
487 - $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
503 + set_transient(
504 + sanitize_key('mxchat_pdf_status_' . md5($pdf_url)),
505 + array_map('sanitize_text_field', $status_data),
506 + DAY_IN_SECONDS
507 + );
488 508
489 - // Store queue ID in transient for status tracking
490 - set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
491 - set_transient('mxchat_last_pdf_url', $pdf_url, DAY_IN_SECONDS);
509 + return __('scheduled', 'mxchat');
492 510
493 - return 'queued';
494 -
495 511 } catch (Exception $e) {
512 + //error_log(sprintf(esc_html__('Error preparing PDF for processing: %s', 'mxchat'), esc_html($e->getMessage())));
496 513 if (file_exists($pdf_path)) {
497 514 wp_delete_file($pdf_path);
498 515 }
499 - return $e->getMessage();
516 + return false;
500 517 }
501 518 }
502 519
503 -/**
504 - * Handle direct PDF file upload from the knowledge base page
505 - */
506 -public function mxchat_handle_pdf_file_submission() {
507 - if (!isset($_POST['submit_pdf_file']) || !current_user_can('manage_options')) {
508 - wp_die(esc_html__('Unauthorized access', 'mxchat'));
509 - }
520 +public function mxchat_process_pdf_pages_cron($pdf_path, $pdf_url, $total_pages, $batch_size, $batch_pause) {
521 + // Validate inputs
522 + $pdf_path = sanitize_text_field($pdf_path);
523 + $pdf_url = esc_url_raw($pdf_url);
524 + $total_pages = absint($total_pages);
525 + $batch_size = absint($batch_size);
526 + $batch_pause = absint($batch_pause);
510 527
511 - check_admin_referer('mxchat_submit_pdf_file_action', 'mxchat_submit_pdf_file_nonce');
512 -
513 - $redirect_url = admin_url('admin.php?page=mxchat-prompts');
514 -
515 - // Validate file upload
516 - if (empty($_FILES['pdf_file']) || $_FILES['pdf_file']['error'] !== UPLOAD_ERR_OK) {
517 - $error_code = isset($_FILES['pdf_file']['error']) ? $_FILES['pdf_file']['error'] : UPLOAD_ERR_NO_FILE;
518 - $error_messages = array(
519 - UPLOAD_ERR_INI_SIZE => __('The uploaded file exceeds the server upload_max_filesize limit.', 'mxchat'),
520 - UPLOAD_ERR_FORM_SIZE => __('The uploaded file exceeds the form MAX_FILE_SIZE limit.', 'mxchat'),
521 - UPLOAD_ERR_PARTIAL => __('The file was only partially uploaded.', 'mxchat'),
522 - UPLOAD_ERR_NO_FILE => __('No file was uploaded. Please select a PDF file.', 'mxchat'),
523 - UPLOAD_ERR_NO_TMP_DIR => __('Server missing temporary folder.', 'mxchat'),
524 - UPLOAD_ERR_CANT_WRITE => __('Server failed to write file to disk.', 'mxchat'),
525 - );
526 - $error_msg = isset($error_messages[$error_code]) ? $error_messages[$error_code] : __('Unknown upload error.', 'mxchat');
527 - set_transient('mxchat_admin_notice_error', $error_msg, 30);
528 - wp_safe_redirect(esc_url($redirect_url));
529 - exit;
530 - }
531 -
532 - $file = $_FILES['pdf_file'];
533 -
534 - // Validate MIME type
535 - $finfo = finfo_open(FILEINFO_MIME_TYPE);
536 - $mime_type = finfo_file($finfo, $file['tmp_name']);
537 - finfo_close($finfo);
538 -
539 - if ($mime_type !== 'application/pdf') {
540 - set_transient('mxchat_admin_notice_error',
541 - esc_html__('Invalid file type. Only PDF files are accepted.', 'mxchat'),
542 - 30
543 - );
544 - wp_safe_redirect(esc_url($redirect_url));
545 - exit;
546 - }
547 -
548 - // Validate extension
549 - $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
550 - if ($ext !== 'pdf') {
551 - set_transient('mxchat_admin_notice_error',
552 - esc_html__('Invalid file extension. Only .pdf files are accepted.', 'mxchat'),
553 - 30
554 - );
555 - wp_safe_redirect(esc_url($redirect_url));
556 - exit;
557 - }
558 -
559 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
560 - $original_filename = sanitize_file_name($file['name']);
561 -
562 - $upload_dir = wp_upload_dir();
563 - if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
564 - set_transient('mxchat_admin_notice_error',
565 - esc_html__('WordPress upload directory is not writable.', 'mxchat'),
566 - 30
567 - );
568 - wp_safe_redirect(esc_url($redirect_url));
569 - exit;
570 - }
571 -
572 - $pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
573 - $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
574 -
575 - if (!wp_mkdir_p(dirname($pdf_path))) {
576 - set_transient('mxchat_admin_notice_error',
577 - esc_html__('Failed to create upload directory.', 'mxchat'),
578 - 30
579 - );
580 - wp_safe_redirect(esc_url($redirect_url));
581 - exit;
582 - }
583 -
584 - // Move uploaded file
585 - if (!move_uploaded_file($file['tmp_name'], $pdf_path)) {
586 - set_transient('mxchat_admin_notice_error',
587 - esc_html__('Failed to save uploaded PDF file.', 'mxchat'),
588 - 30
589 - );
590 - wp_safe_redirect(esc_url($redirect_url));
591 - exit;
592 - }
593 -
594 528 try {
595 - $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
596 -
597 - if ($total_pages === false || $total_pages < 1) {
598 - throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
529 + if (!file_exists($pdf_path)) {
530 + throw new Exception(sprintf('PDF file not found at path: %s', esc_html($pdf_path)));
599 531 }
600 532
601 - // Use original filename as the source identifier
602 - $source_label = 'upload://' . $original_filename;
533 + $parser = new \Smalot\PdfParser\Parser();
534 + $pdf = $parser->parseFile($pdf_path);
535 + $pages = $pdf->getPages();
603 536
604 - $queue_id = 'pdf_' . md5($source_label . time());
537 + // Get current progress
538 + $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
539 + $status = get_transient($status_key);
605 540
606 - $pages = array();
607 - for ($i = 1; $i <= $total_pages; $i++) {
608 - $pages[] = array(
609 - 'pdf_path' => $pdf_path,
610 - 'pdf_url' => $source_label,
611 - 'page_number' => $i,
612 - 'total_pages' => $total_pages,
613 - );
541 + if (!$status || !is_array($status)) {
542 + throw new Exception('Invalid status data retrieved from transient');
614 543 }
615 544
616 - $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
545 + // Initialize failed pages list if it doesn't exist
546 + if (!isset($status['failed_pages_list']) || !is_array($status['failed_pages_list'])) {
547 + $status['failed_pages_list'] = [];
548 + }
617 549
618 - if ($queued_count === 0) {
619 - wp_delete_file($pdf_path);
620 - throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
550 + $start_page = absint($status['processed_pages']);
551 + $end_page = min($start_page + $batch_size, $total_pages);
552 + $options = get_option('mxchat_options');
553 +
554 + if (empty($options['api_key'])) {
555 + throw new Exception('API key is missing or invalid');
621 556 }
622 557
623 - $this->mxchat_set_queue_meta($queue_id, 'source_url', $source_label);
624 - $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
625 - $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
626 - $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
627 - $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
628 - $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
558 + $successful_pages = 0;
559 + $failed_pages = 0;
629 560
630 - set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
631 - set_transient('mxchat_last_pdf_url', $source_label, DAY_IN_SECONDS);
561 + for ($i = $start_page; $i < $end_page; $i++) {
562 + $page_number = $i + 1;
563 + $max_retries = 3;
564 + $retry_count = 0;
565 + $page_processed = false;
566 + $last_error = '';
632 567
633 - set_transient('mxchat_admin_notice_success',
634 - sprintf(
635 - esc_html__('PDF "%s" (%d pages) queued for processing. Processing will start automatically.', 'mxchat'),
636 - esc_html($original_filename),
637 - $total_pages
638 - ),
639 - 30
640 - );
568 + while (!$page_processed && $retry_count < $max_retries) {
569 + try {
570 + $text = $pages[$i]->getText();
571 +
572 + if (empty($text)) {
573 + throw new Exception("Empty text on page {$page_number}");
574 + }
575 +
576 + $sanitized_content = $this->mxchat_sanitize_content_for_api($text);
641 577
642 - } catch (Exception $e) {
643 - if (file_exists($pdf_path)) {
644 - wp_delete_file($pdf_path);
645 - }
646 - set_transient('mxchat_admin_notice_error',
647 - esc_html__('Failed to process uploaded PDF: ', 'mxchat') . esc_html($e->getMessage()),
648 - 30
649 - );
650 - }
578 + if (empty($sanitized_content)) {
579 + throw new Exception("No valid content after sanitization on page {$page_number}");
580 + }
651 581
652 - wp_safe_redirect(esc_url($redirect_url));
653 - exit;
654 -}
582 + $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
583 +
584 + if (is_string($embedding_vector)) {
585 + throw new Exception("Embedding generation failed: " . $embedding_vector);
586 + }
587 +
588 + if (!is_array($embedding_vector)) {
589 + throw new Exception("Embedding generation returned unexpected result type: " . gettype($embedding_vector));
590 + }
591 +
592 + $metadata = array(
593 + 'document_type' => 'pdf',
594 + 'total_pages' => $total_pages,
595 + 'current_page' => $page_number,
596 + 'prev_page' => $i > 0 ? $i : null,
597 + 'next_page' => $i < ($total_pages - 1) ? $i + 2 : null,
598 + 'source_url' => $pdf_url
599 + );
655 600
656 -/**
657 - * Validate PDF and count pages with multiple parser attempts
658 - */
659 -private function mxchat_validate_and_count_pdf_pages($pdf_path) {
660 - // Method 1: Try with Smalot PDF Parser (your current method)
661 - try {
662 - mxchat_load_pdf_parser();
663 - $parser = new \Smalot\PdfParser\Parser();
664 - $pdf = $parser->parseFile($pdf_path);
665 - $pages = $pdf->getPages();
666 - $page_count = count($pages);
667 -
668 - if ($page_count > 0) {
669 - //error_log('PDF parsed successfully with Smalot parser: ' . $page_count . ' pages');
670 - return $page_count;
671 - }
672 - } catch (Exception $e) {
673 - //error_log('Smalot PDF parser failed: ' . $e->getMessage());
674 - }
601 + $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized_content;
602 + $page_url = esc_url($pdf_url . "#page=" . $page_number);
675 603
676 - // Method 2: Try with pdfinfo command (if available)
677 - if (function_exists('shell_exec') && !$this->mxchat_is_shell_disabled()) {
678 - try {
679 - $command = 'pdfinfo ' . escapeshellarg($pdf_path) . ' 2>&1';
680 - $output = shell_exec($command);
681 -
682 - if ($output && preg_match('/Pages:\s*(\d+)/', $output, $matches)) {
683 - $page_count = intval($matches[1]);
684 - if ($page_count > 0) {
685 - //error_log('PDF parsed successfully with pdfinfo: ' . $page_count . ' pages');
686 - return $page_count;
604 + $db_result = MxChat_Utils::submit_content_to_db($content_with_metadata, $page_url, $options['api_key']);
605 +
606 + if (is_wp_error($db_result)) {
607 + throw new Exception("Database submission failed: " . $db_result->get_error_message());
608 + }
609 +
610 + // Success!
611 + $page_processed = true;
612 + $successful_pages++;
613 +
614 + } catch (Exception $e) {
615 + $retry_count++;
616 + $last_error = $e->getMessage();
617 +
618 + //error_log("PDF page {$page_number} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
619 +
620 + if ($retry_count < $max_retries) {
621 + // Wait before retry (exponential backoff: 1s, 2s, 4s)
622 + sleep(pow(2, $retry_count - 1));
623 + }
687 624 }
688 625 }
689 - } catch (Exception $e) {
690 - //error_log('pdfinfo command failed: ' . $e->getMessage());
691 - }
692 - }
693 626
694 - // Method 3: Try to repair PDF and parse again
695 - try {
696 - $repaired_path = $this->mxchat_attempt_pdf_repair($pdf_path);
697 - if ($repaired_path && $repaired_path !== $pdf_path) {
698 - mxchat_load_pdf_parser();
699 - $parser = new \Smalot\PdfParser\Parser();
700 - $pdf = $parser->parseFile($repaired_path);
701 - $pages = $pdf->getPages();
702 - $page_count = count($pages);
703 -
704 - if ($page_count > 0) {
705 - // Replace original with repaired version
706 - copy($repaired_path, $pdf_path);
707 - unlink($repaired_path);
708 - //error_log('PDF repaired and parsed successfully: ' . $page_count . ' pages');
709 - return $page_count;
627 + // If page still not processed after all retries, mark as failed
628 + if (!$page_processed) {
629 + $failed_pages++;
630 + $status['failed_pages_list'][] = [
631 + 'page' => $page_number,
632 + 'error' => $last_error,
633 + 'time' => time(),
634 + 'retries' => $max_retries
635 + ];
636 +
637 + // Limit failed pages list to prevent memory issues
638 + if (count($status['failed_pages_list']) > 50) {
639 + $status['failed_pages_list'] = array_slice($status['failed_pages_list'], -50);
640 + }
641 +
642 + //error_log("PDF page {$page_number} permanently failed after {$max_retries} attempts: " . $last_error);
710 643 }
644 +
645 + // Update progress
646 + $status['processed_pages'] = absint($page_number);
647 + $status['last_update'] = time();
648 + $status['failed_pages'] = absint($status['failed_pages'] ?? 0) + ($page_processed ? 0 : 1);
711 649
712 - // Clean up repaired file if it didn't work
713 - unlink($repaired_path);
650 + set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
714 651 }
715 - } catch (Exception $e) {
716 - //error_log('PDF repair attempt failed: ' . $e->getMessage());
717 - }
718 652
719 - // Method 4: Manual PDF structure analysis (basic page count)
720 - try {
721 - $page_count = $this->mxchat_manual_pdf_page_count($pdf_path);
722 - if ($page_count > 0) {
723 - //error_log('PDF page count determined manually: ' . $page_count . ' pages');
724 - return $page_count;
725 - }
726 - } catch (Exception $e) {
727 - //error_log('Manual PDF analysis failed: ' . $e->getMessage());
728 - }
729 -
730 - //error_log('All PDF parsing methods failed for: ' . $pdf_path);
731 - return false;
732 -}
733 -
734 -/**
735 - * Check if shell_exec is disabled
736 - */
737 -private function mxchat_is_shell_disabled() {
738 - $disabled = explode(',', ini_get('disable_functions'));
739 - return in_array('shell_exec', $disabled);
740 -}
741 -
742 -/**
743 - * Attempt to repair PDF using basic methods
744 - */
745 -private function mxchat_attempt_pdf_repair($pdf_path) {
746 - try {
747 - $content = file_get_contents($pdf_path);
748 - if (!$content) {
749 - return false;
750 - }
751 -
752 - // Check if PDF starts with proper header
753 - if (substr($content, 0, 4) !== '%PDF') {
754 - // Try to find PDF header in the content
755 - $header_pos = strpos($content, '%PDF');
756 - if ($header_pos !== false && $header_pos < 1024) {
757 - // Remove junk before PDF header
758 - $content = substr($content, $header_pos);
759 - $repaired_path = $pdf_path . '.repaired';
760 - file_put_contents($repaired_path, $content);
761 - return $repaired_path;
653 + // Schedule next batch if needed
654 + if ($end_page < $total_pages) {
655 + wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_pdf_pages', array(
656 + 'pdf_path' => $pdf_path,
657 + 'pdf_url' => $pdf_url,
658 + 'total_pages' => $total_pages,
659 + 'batch_size' => $batch_size,
660 + 'batch_pause' => $batch_pause
661 + ));
662 + } else {
663 + // Processing complete
664 + $status['status'] = 'complete';
665 + $status['processed_pages'] = $total_pages;
666 +
667 + // Add completion summary
668 + $status['completion_summary'] = [
669 + 'total_pages' => $total_pages,
670 + 'successful_pages' => $total_pages - absint($status['failed_pages'] ?? 0),
671 + 'failed_pages' => absint($status['failed_pages'] ?? 0),
672 + 'completion_time' => current_time('mysql')
673 + ];
674 +
675 + // Save the completed status (don't delete it - let user dismiss manually)
676 + set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
677 +
678 + // Clean up the temporary PDF file
679 + if (file_exists($pdf_path)) {
680 + wp_delete_file($pdf_path);
762 681 }
682 +
683 + // DON'T delete the status transients here - let user dismiss manually
763 684 }
764 685
765 - // Check for EOF marker
766 - $content = rtrim($content);
767 - if (!preg_match('/%%EOF\s*$/', $content)) {
768 - // Add EOF marker if missing
769 - $content .= "\n%%EOF";
770 - $repaired_path = $pdf_path . '.repaired';
771 - file_put_contents($repaired_path, $content);
772 - return $repaired_path;
686 + } catch (\Exception $e) {
687 + //error_log(sprintf('[MXCHAT-PDF] Error processing PDF: %s', $e->getMessage()));
688 +
689 + $status_key = sanitize_key('mxchat_pdf_status_' . md5($pdf_url));
690 + $status = get_transient($status_key);
691 +
692 + if (!$status || !is_array($status)) {
693 + $status = array(
694 + 'total_pages' => $total_pages,
695 + 'processed_pages' => 0,
696 + 'status' => 'error',
697 + 'error' => sanitize_text_field($e->getMessage()),
698 + 'last_update' => time()
699 + );
700 + } else {
701 + $status['status'] = 'error';
702 + $status['error'] = sanitize_text_field($e->getMessage());
703 + $status['last_update'] = time();
773 704 }
774 -
775 - } catch (Exception $e) {
776 - //error_log('PDF repair error: ' . $e->getMessage());
777 - }
778 -
779 - return false;
780 -}
781 -
782 -/**
783 - * Manual PDF page counting by analyzing PDF structure
784 - */
785 -private function mxchat_manual_pdf_page_count($pdf_path) {
786 - try {
787 - $content = file_get_contents($pdf_path);
788 - if (!$content) {
789 - return 0;
705 +
706 + set_transient($status_key, array_map('sanitize_text_field', $status), DAY_IN_SECONDS);
707 +
708 + if (file_exists($pdf_path)) {
709 + wp_delete_file($pdf_path);
790 710 }
791 -
792 - // Method 1: Count /Type /Page objects
793 - $page_count = preg_match_all('/\/Type\s*\/Page[^s]/', $content);
794 - if ($page_count > 0) {
795 - return $page_count;
796 - }
797 -
798 - // Method 2: Look for /Count in pages object
799 - if (preg_match('/\/Type\s*\/Pages.*?\/Count\s+(\d+)/', $content, $matches)) {
800 - return intval($matches[1]);
801 - }
802 -
803 - // Method 3: Count page references
804 - $page_count = preg_match_all('/\d+\s+0\s+obj\s*<<[^>]*\/Type\s*\/Page/', $content);
805 - if ($page_count > 0) {
806 - return $page_count;
807 - }
808 -
809 - } catch (Exception $e) {
810 - //error_log('Manual PDF analysis error: ' . $e->getMessage());
811 711 }
812 -
813 - return 0;
814 712 }
815 713
816 -
817 714 public function mxchat_save_inline_prompt() {
818 715 // DEBUG: Log what we're receiving
819 - //error_log('=== MXCHAT DEBUG ===');
820 - //error_log('POST data: ' . print_r($_POST, true));
821 - //error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
716 + error_log('=== MXCHAT DEBUG ===');
717 + error_log('POST data: ' . print_r($_POST, true));
718 + error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
822 719
823 720 // Check for nonce security
824 721 check_ajax_referer('mxchat_save_inline_nonce', '_ajax_nonce');
825 722
826 723 // If we get here, nonce passed
827 - //error_log('Nonce verification PASSED');
828 -
829 - // Verify permissions
830 - if (!current_user_can('manage_options')) {
831 - wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
832 - return;
833 - }
834 -
835 - global $wpdb;
836 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
837 -
838 - // Validate and sanitize input data
839 - $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
840 - $article_content = isset($_POST['article_content']) ? wp_kses_post(wp_unslash($_POST['article_content'])) : '';
841 - $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
842 -
843 - if ($prompt_id > 0 && !empty($article_content)) {
844 - // Re-generate the embedding vector for the updated content
845 - $embedding_vector = $this->mxchat_generate_embedding($article_content);
846 - if (is_array($embedding_vector)) {
847 - // Serialize the embedding vector before storing it
848 - $embedding_vector_serialized = serialize($embedding_vector);
849 - // Update the prompt in the database
850 - $updated = $wpdb->update(
851 - $table_name,
852 - array(
853 - 'article_content' => $article_content,
854 - 'embedding_vector' => $embedding_vector_serialized,
855 - 'source_url' => $article_url,
856 - ),
857 - array('id' => $prompt_id),
858 - array('%s', '%s', '%s'),
859 - array('%d')
860 - );
861 - if ($updated !== false) {
862 - wp_send_json_success();
863 - } else {
864 - MxChat_Admin::mxchat_log_debug('knowledge_error', 'Database update failed when saving knowledge entry');
865 - wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
866 - }
867 - } else {
868 - MxChat_Admin::mxchat_log_debug('embedding_error', 'Embedding generation failed for knowledge entry');
869 - wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
870 - }
871 - } else {
872 - wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
873 - }
874 -}
724 + error_log('Nonce verification PASSED');
875 725
726 + // Verify permissions
727 + if (!current_user_can('manage_options')) {
728 + wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
729 + return;
730 + }
876 731
877 -/**
878 - * AJAX: Get full content for editing — reassembles chunks if needed.
879 - * Works for both WordPress DB and Pinecone entries.
880 - */
881 -public function ajax_mxchat_get_entry_content() {
882 - check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
732 + global $wpdb;
733 + $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
883 734
884 - if ( ! current_user_can('manage_options') ) {
885 - wp_send_json_error( array( 'message' => 'Permission denied.' ) );
886 - }
735 + // Validate and sanitize input data
736 + $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
737 + $article_content = isset($_POST['article_content']) ? sanitize_textarea_field($_POST['article_content']) : '';
738 + $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
887 739
888 - $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
889 - $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
890 - $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
891 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
740 + if ($prompt_id > 0 && !empty($article_content)) {
741 + // Re-generate the embedding vector for the updated content
742 + $embedding_vector = $this->mxchat_generate_embedding($article_content);
892 743
893 - if ( $data_source === 'pinecone' ) {
894 - // Pinecone: fetch vectors by source_url, reassemble chunks
895 - $content = $this->get_pinecone_entry_content( $source_url, $entry_id, $bot_id );
896 - } else {
897 - // WordPress DB
898 - $content = $this->get_wordpress_entry_content( $source_url, $entry_id );
899 - }
744 + if (is_array($embedding_vector)) {
745 + // Serialize the embedding vector before storing it
746 + $embedding_vector_serialized = serialize($embedding_vector);
900 747
901 - if ( is_wp_error( $content ) ) {
902 - wp_send_json_error( array( 'message' => $content->get_error_message() ) );
903 - }
748 + // Update the prompt in the database
749 + $updated = $wpdb->update(
750 + $table_name,
751 + array(
752 + 'article_content' => $article_content,
753 + 'embedding_vector' => $embedding_vector_serialized,
754 + 'source_url' => $article_url,
755 + ),
756 + array('id' => $prompt_id),
757 + array('%s', '%s', '%s'),
758 + array('%d')
759 + );
904 760
905 - wp_send_json_success( $content );
906 -}
761 + if ($updated !== false) {
762 + wp_send_json_success();
763 + } else {
764 + wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
765 + }
766 + } else {
767 + wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
768 + }
769 + } else {
770 + wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
771 + }
772 + }
907 773
908 -/**
909 - * Get content from WordPress DB — reassembles chunks by source_url.
910 - */
911 -private function get_wordpress_entry_content( $source_url, $entry_id ) {
912 - global $wpdb;
913 - $table = $wpdb->prefix . 'mxchat_system_prompt_content';
914 774
915 - // If we have a source_url, check for chunks
916 - if ( ! empty( $source_url ) && strpos( $source_url, 'mxchat://' ) !== 0 ) {
917 - $rows = $wpdb->get_results( $wpdb->prepare(
918 - "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
919 - $source_url
920 - ) );
921 -
922 - if ( $rows && count( $rows ) > 1 ) {
923 - // Multiple rows = chunked. Reassemble.
924 - $chunks = array();
925 - foreach ( $rows as $row ) {
926 - $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
927 - $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
928 - $chunks[ $index ] = $parsed['text'];
929 - }
930 - ksort( $chunks );
931 - return array(
932 - 'content' => implode( "\n\n", $chunks ),
933 - 'source_url' => $source_url,
934 - 'is_chunked' => true,
935 - 'chunk_count' => count( $chunks ),
936 - 'content_type' => $rows[0]->content_type,
937 - );
938 - } elseif ( $rows && count( $rows ) === 1 ) {
939 - $parsed = MxChat_Chunker::parse_stored_chunk( $rows[0]->article_content );
940 - return array(
941 - 'content' => $parsed['text'],
942 - 'source_url' => $source_url,
943 - 'entry_id' => $rows[0]->id,
944 - 'is_chunked' => false,
945 - 'content_type' => $rows[0]->content_type,
946 - );
947 - }
948 - }
949 -
950 - // Fallback: fetch by ID
951 - if ( $entry_id > 0 ) {
952 - $row = $wpdb->get_row( $wpdb->prepare(
953 - "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
954 - $entry_id
955 - ) );
956 - if ( $row ) {
957 - $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
958 - return array(
959 - 'content' => $parsed['text'],
960 - 'source_url' => $row->source_url,
961 - 'entry_id' => $row->id,
962 - 'is_chunked' => false,
963 - 'content_type' => $row->content_type,
964 - );
965 - }
966 - }
967 -
968 - return new WP_Error( 'not_found', 'Entry not found.' );
969 -}
970 -
971 -/**
972 - * Get content from Pinecone — fetches vectors by source_url, reassembles chunks.
973 - */
974 -private function get_pinecone_entry_content( $source_url, $entry_id, $bot_id ) {
975 - if ( ! class_exists('MxChat_Pinecone_Manager') ) {
976 - return new WP_Error( 'pinecone_unavailable', 'Pinecone manager not available.' );
977 - }
978 -
979 - // Get Pinecone config
980 - if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
981 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
982 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
983 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
984 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
985 - } else {
986 - $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
987 - $api_key = $bot_config['api_key'] ?? '';
988 - $host = $bot_config['host'] ?? '';
989 - $namespace = $bot_config['namespace'] ?? '';
990 - }
991 -
992 - if ( empty($host) || empty($api_key) ) {
993 - return new WP_Error( 'pinecone_config', 'Pinecone not configured.' );
994 - }
995 -
996 - // List vectors with the source_url prefix
997 - $base_id = md5( $source_url );
998 - $vector_ids = array( $base_id );
999 -
1000 - // Find chunk vectors
1001 - $list_url = "https://{$host}/vectors/list";
1002 - $list_body = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
1003 - if ( ! empty($namespace) ) {
1004 - $list_body['namespace'] = $namespace;
1005 - }
1006 -
1007 - $list_resp = wp_remote_post( $list_url, array(
1008 - 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1009 - 'body' => wp_json_encode( $list_body ),
1010 - 'timeout' => 15,
1011 - ) );
1012 -
1013 - if ( ! is_wp_error($list_resp) ) {
1014 - $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
1015 - if ( ! empty($list_data['vectors']) ) {
1016 - foreach ( $list_data['vectors'] as $v ) {
1017 - $vector_ids[] = $v['id'];
1018 - }
1019 - }
1020 - }
1021 -
1022 - // Fetch vectors with metadata
1023 - $fetch_url = "https://{$host}/vectors/fetch";
1024 - $fetch_body = array( 'ids' => $vector_ids );
1025 - if ( ! empty($namespace) ) {
1026 - $fetch_body['namespace'] = $namespace;
1027 - }
1028 -
1029 - $fetch_resp = wp_remote_post( $fetch_url, array(
1030 - 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1031 - 'body' => wp_json_encode( $fetch_body ),
1032 - 'timeout' => 15,
1033 - ) );
1034 -
1035 - if ( is_wp_error($fetch_resp) ) {
1036 - return new WP_Error( 'pinecone_fetch', 'Failed to fetch from Pinecone.' );
1037 - }
1038 -
1039 - $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
1040 - $vectors = $fetch_data['vectors'] ?? array();
1041 -
1042 - if ( empty($vectors) ) {
1043 - return new WP_Error( 'not_found', 'Entry not found in Pinecone.' );
1044 - }
1045 -
1046 - // Reassemble chunks
1047 - $chunks = array();
1048 - $content_type = 'content';
1049 - foreach ( $vectors as $vid => $vector ) {
1050 - $meta = $vector['metadata'] ?? array();
1051 - $text = $meta['text'] ?? '';
1052 - $index = $meta['chunk_index'] ?? 0;
1053 - $content_type = $meta['type'] ?? 'content';
1054 - $chunks[ intval($index) ] = $text;
1055 - }
1056 - ksort( $chunks );
1057 -
1058 - return array(
1059 - 'content' => implode( "\n\n", $chunks ),
1060 - 'source_url' => $source_url,
1061 - 'is_chunked' => count($chunks) > 1,
1062 - 'chunk_count' => count($chunks),
1063 - 'content_type' => $content_type,
1064 - );
1065 -}
1066 -
1067 -/**
1068 - * AJAX: Inspect a knowledge entry — returns the per-chunk stored text + metadata
1069 - * WITHOUT collapsing it, so a site owner can see exactly what was indexed for an
1070 - * entry (plan-mxchat-20260628-d8cb4b). READ-ONLY: never re-embeds or mutates.
1071 - */
1072 -public function ajax_mxchat_inspect_entry() {
1073 - check_ajax_referer('mxchat_inspect_entry_nonce', 'nonce');
1074 -
1075 - if ( ! current_user_can('manage_options') ) {
1076 - wp_send_json_error( array( 'message' => esc_html__('Permission denied.', 'mxchat') ) );
1077 - }
1078 -
1079 - $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
1080 - $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
1081 - $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
1082 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1083 -
1084 - if ( $data_source === 'pinecone' ) {
1085 - $result = $this->inspect_pinecone_entry( $source_url, $entry_id, $bot_id );
1086 - } else {
1087 - $result = $this->inspect_wordpress_entry( $source_url, $entry_id );
1088 - }
1089 -
1090 - if ( is_wp_error( $result ) ) {
1091 - wp_send_json_error( array( 'message' => $result->get_error_message() ) );
1092 - }
1093 -
1094 - wp_send_json_success( $result );
1095 -}
1096 -
1097 -/**
1098 - * Read-only inspector for WordPress-DB entries. Mirrors get_wordpress_entry_content()
1099 - * but returns each STORED chunk's exact text + length (no implode), plus the assembled
1100 - * embedded text. This shows what is actually in the index, not a re-derivation from the post.
1101 - */
1102 -private function inspect_wordpress_entry( $source_url, $entry_id ) {
1103 - global $wpdb;
1104 - $table = $wpdb->prefix . 'mxchat_system_prompt_content';
1105 -
1106 - $rows = array();
1107 -
1108 - // Group by the real stored source_url — this INCLUDES "mxchat://" manual
1109 - // Direct Content entries (the spec's manual-entry case), which share one
1110 - // source_url across their chunk rows. Only the synthetic "_ungrouped_<id>"
1111 - // display key (invented by the table view for rows with no source_url) is
1112 - // excluded; those fall through to the entry_id lookup below.
1113 - if ( ! empty( $source_url ) && strpos( $source_url, '_ungrouped_' ) !== 0 ) {
1114 - $rows = $wpdb->get_results( $wpdb->prepare(
1115 - "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
1116 - $source_url
1117 - ) );
1118 - }
1119 -
1120 - // Fallback / manual "Direct Content" entries: fetch the single row by id.
1121 - if ( empty( $rows ) && $entry_id > 0 ) {
1122 - $row = $wpdb->get_row( $wpdb->prepare(
1123 - "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
1124 - $entry_id
1125 - ) );
1126 - if ( $row ) {
1127 - $rows = array( $row );
1128 - }
1129 - }
1130 -
1131 - if ( empty( $rows ) ) {
1132 - return new WP_Error( 'not_found', esc_html__('Entry not found in the local knowledge database.', 'mxchat') );
1133 - }
1134 -
1135 - $chunks = array();
1136 - $content_type = '';
1137 - foreach ( $rows as $row ) {
1138 - $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
1139 - $text = isset( $parsed['text'] ) ? $parsed['text'] : '';
1140 - $index = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
1141 - $content_type = $row->content_type;
1142 - $chunks[] = array(
1143 - 'index' => $index,
1144 - 'text' => $text,
1145 - 'length' => function_exists('mb_strlen') ? mb_strlen( $text ) : strlen( $text ),
1146 - 'row_id' => intval( $row->id ),
1147 - );
1148 - }
1149 -
1150 - usort( $chunks, function( $a, $b ) { return $a['index'] - $b['index']; } );
1151 -
1152 - $assembled = implode( "\n\n", wp_list_pluck( $chunks, 'text' ) );
1153 -
1154 - return array(
1155 - 'store' => 'wordpress',
1156 - 'source_url' => $source_url,
1157 - 'content_type' => $content_type,
1158 - 'is_chunked' => count( $chunks ) > 1,
1159 - 'chunk_count' => count( $chunks ),
1160 - 'assembled' => $assembled,
1161 - 'assembled_length' => function_exists('mb_strlen') ? mb_strlen( $assembled ) : strlen( $assembled ),
1162 - 'chunks' => array_values( $chunks ),
1163 - // WP-DB storage carries no separate vector metadata; surface that fact
1164 - // rather than letting the owner guess (the spec's taxonomy question).
1165 - 'metadata' => array(),
1166 - 'metadata_note' => esc_html__('Stored in the local WordPress database. Only the assembled text shown here is embedded — there are no separate vector metadata fields (e.g. taxonomy terms are not stored unless they were injected into the text itself).', 'mxchat'),
1167 - );
1168 -}
1169 -
1170 -/**
1171 - * Read-only inspector for Pinecone entries. Mirrors get_pinecone_entry_content()
1172 - * but keeps each vector's text + metadata instead of imploding, so the owner can
1173 - * confirm exactly which metadata fields (text/source_url/type/last_updated/created_at/bot_id)
1174 - * are present per chunk. READ-ONLY.
1175 - */
1176 -private function inspect_pinecone_entry( $source_url, $entry_id, $bot_id ) {
1177 - if ( ! class_exists('MxChat_Pinecone_Manager') ) {
1178 - return new WP_Error( 'pinecone_unavailable', esc_html__('Pinecone manager not available.', 'mxchat') );
1179 - }
1180 -
1181 - if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
1182 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
1183 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1184 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1185 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1186 - } else {
1187 - $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1188 - $api_key = $bot_config['api_key'] ?? '';
1189 - $host = $bot_config['host'] ?? '';
1190 - $namespace = $bot_config['namespace'] ?? '';
1191 - }
1192 -
1193 - if ( empty($host) || empty($api_key) ) {
1194 - return new WP_Error( 'pinecone_config', esc_html__('Pinecone not configured.', 'mxchat') );
1195 - }
1196 -
1197 - $base_id = md5( $source_url );
1198 - $vector_ids = array( $base_id );
1199 -
1200 - $list_url = "https://{$host}/vectors/list";
1201 - $list_body = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
1202 - if ( ! empty($namespace) ) {
1203 - $list_body['namespace'] = $namespace;
1204 - }
1205 -
1206 - $list_resp = wp_remote_post( $list_url, array(
1207 - 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1208 - 'body' => wp_json_encode( $list_body ),
1209 - 'timeout' => 15,
1210 - ) );
1211 -
1212 - if ( ! is_wp_error($list_resp) ) {
1213 - $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
1214 - if ( ! empty($list_data['vectors']) ) {
1215 - foreach ( $list_data['vectors'] as $v ) {
1216 - $vector_ids[] = $v['id'];
1217 - }
1218 - }
1219 - }
1220 -
1221 - $fetch_url = "https://{$host}/vectors/fetch";
1222 - $fetch_body = array( 'ids' => $vector_ids );
1223 - if ( ! empty($namespace) ) {
1224 - $fetch_body['namespace'] = $namespace;
1225 - }
1226 -
1227 - $fetch_resp = wp_remote_post( $fetch_url, array(
1228 - 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ),
1229 - 'body' => wp_json_encode( $fetch_body ),
1230 - 'timeout' => 15,
1231 - ) );
1232 -
1233 - if ( is_wp_error($fetch_resp) ) {
1234 - return new WP_Error( 'pinecone_fetch', esc_html__('Failed to fetch from Pinecone.', 'mxchat') );
1235 - }
1236 -
1237 - $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
1238 - $vectors = $fetch_data['vectors'] ?? array();
1239 -
1240 - if ( empty($vectors) ) {
1241 - return new WP_Error( 'not_found', esc_html__('Entry not found in Pinecone.', 'mxchat') );
1242 - }
1243 -
1244 - // Whitelisted metadata fields the spec calls out — shown so devs can confirm
1245 - // what is (and is NOT) stored per vector.
1246 - $meta_fields = array( 'text', 'source_url', 'type', 'last_updated', 'created_at', 'bot_id', 'chunk_index', 'total_chunks' );
1247 - $chunks = array();
1248 - $content_type = '';
1249 - foreach ( $vectors as $vid => $vector ) {
1250 - $meta = isset($vector['metadata']) && is_array($vector['metadata']) ? $vector['metadata'] : array();
1251 - $text = $meta['text'] ?? '';
1252 - $index = isset($meta['chunk_index']) ? intval($meta['chunk_index']) : count($chunks);
1253 - $content_type = $meta['type'] ?? $content_type;
1254 -
1255 - $clean_meta = array();
1256 - foreach ( $meta_fields as $field ) {
1257 - if ( array_key_exists( $field, $meta ) && $field !== 'text' ) {
1258 - $clean_meta[ $field ] = is_scalar( $meta[ $field ] ) ? (string) $meta[ $field ] : wp_json_encode( $meta[ $field ] );
1259 - }
1260 - }
1261 -
1262 - $chunks[] = array(
1263 - 'index' => $index,
1264 - 'text' => $text,
1265 - 'length' => function_exists('mb_strlen') ? mb_strlen( $text ) : strlen( $text ),
1266 - 'vector_id' => (string) $vid,
1267 - 'metadata' => $clean_meta,
1268 - );
1269 - }
1270 -
1271 - usort( $chunks, function( $a, $b ) { return $a['index'] - $b['index']; } );
1272 -
1273 - $assembled = implode( "\n\n", wp_list_pluck( $chunks, 'text' ) );
1274 -
1275 - return array(
1276 - 'store' => 'pinecone',
1277 - 'source_url' => $source_url,
1278 - 'content_type' => $content_type,
1279 - 'is_chunked' => count( $chunks ) > 1,
1280 - 'chunk_count' => count( $chunks ),
1281 - 'assembled' => $assembled,
1282 - 'assembled_length' => function_exists('mb_strlen') ? mb_strlen( $assembled ) : strlen( $assembled ),
1283 - 'chunks' => array_values( $chunks ),
1284 - 'metadata' => array(),
1285 - 'metadata_note' => esc_html__('Stored in Pinecone. Each chunk above lists the vector metadata fields actually present — if a field you expect (such as taxonomy terms) is missing here, it was not stored as metadata and is only searchable if it appears in the embedded text.', 'mxchat'),
1286 - );
1287 -}
1288 -
1289 -/**
1290 - * AJAX: Save edited content — re-chunks and re-embeds as needed.
1291 - * Works for both WordPress DB and Pinecone entries.
1292 - */
1293 -public function ajax_mxchat_save_entry_content() {
1294 - check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');
1295 -
1296 - if ( ! current_user_can('manage_options') ) {
1297 - wp_send_json_error( array( 'message' => 'Permission denied.' ) );
1298 - }
1299 -
1300 - $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
1301 - $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
1302 - $content = isset($_POST['content']) ? wp_kses_post( wp_unslash($_POST['content']) ) : '';
1303 - $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
1304 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1305 - $content_type = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : 'content';
1306 -
1307 - if ( empty($content) ) {
1308 - wp_send_json_error( array( 'message' => 'Content cannot be empty.' ) );
1309 - }
1310 -
1311 - // Get the embedding API key
1312 - $options = get_option('mxchat_options', array());
1313 - $api_key = '';
1314 -
1315 - if ( $bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager') ) {
1316 - $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
1317 - $api_key = $bot_options['api_key'] ?? '';
1318 - }
1319 - if ( empty($api_key) ) {
1320 - $api_key = $options['api_key'] ?? '';
1321 - }
1322 -
1323 - global $wpdb;
1324 - $table = $wpdb->prefix . 'mxchat_system_prompt_content';
1325 -
1326 - // If source_url is empty but we have an entry_id, look it up
1327 - if ( empty($source_url) && $entry_id > 0 && $data_source === 'wordpress' ) {
1328 - $row = $wpdb->get_row( $wpdb->prepare( "SELECT source_url FROM {$table} WHERE id = %d", $entry_id ) );
1329 - if ( $row && ! empty($row->source_url) ) {
1330 - $source_url = $row->source_url;
1331 - }
1332 - }
1333 -
1334 - // For manual entries (no source_url or mxchat:// prefix), delete the old entry by ID first
1335 - // so submit_content_to_db creates a replacement instead of a duplicate
1336 - // Also treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
1337 - $is_legacy_manual = !empty($source_url) && strpos($source_url, 'mxchat.ai') !== false && strpos($source_url, 'mxchat://') !== 0;
1338 - if ( $entry_id > 0 && (empty($source_url) || strpos($source_url, 'mxchat://') === 0 || $is_legacy_manual) ) {
1339 - $wpdb->delete( $table, array( 'id' => $entry_id ), array( '%d' ) );
1340 - // Clear legacy URL so submit_content_to_db generates a unique mxchat:// identifier
1341 - // instead of reusing the shared URL (which would mass-delete other entries with the same URL)
1342 - if ( $is_legacy_manual ) {
1343 - $source_url = '';
1344 - }
1345 - }
1346 -
1347 - // Use the existing submit_content_to_db which handles chunking, Pinecone, and WP DB
1348 - $vector_id = ! empty($source_url) ? md5($source_url) : md5('mxchat_manual_' . $entry_id);
1349 -
1350 - // submit_content_to_db already handles: delete old chunks → re-chunk → re-embed → store
1351 - $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, $vector_id, $bot_id, $content_type );
1352 -
1353 - if ( is_wp_error($result) ) {
1354 - wp_send_json_error( array( 'message' => $result->get_error_message() ) );
1355 - }
1356 -
1357 - wp_send_json_success( array( 'message' => 'Content saved and re-embedded successfully.' ) );
1358 -}
1359 -
1360 775 public function mxchat_get_pdf_processing_status($pdf_url) {
1361 776 $pdf_url = esc_url_raw($pdf_url);
1362 777 $status = get_transient(sanitize_key('mxchat_pdf_status_' . md5($pdf_url)));
1363 778
@@ -1400,18 +815,24 @@
1400 815 }
1401 816
1402 817
1403 818 public function mxchat_handle_sitemap_submission() {
819 + // Start logging the submission process
820 + //error_log('[MXCHAT-URL] ===== Starting URL submission process =====');
821 +
1404 822 // Check if the form was submitted and verify permissions
1405 823 if (!isset($_POST['submit_sitemap']) || !current_user_can('manage_options')) {
824 + //error_log('[MXCHAT-URL] Error: Unauthorized access or form not submitted properly');
1406 825 wp_die(esc_html__('Unauthorized access', 'mxchat'));
1407 826 }
1408 827
1409 828 // Verify nonce
829 + //error_log('[MXCHAT-URL] Verifying nonce');
1410 830 check_admin_referer('mxchat_submit_sitemap_action', 'mxchat_submit_sitemap_nonce');
1411 831
1412 832 // Validate URL
1413 833 if (!isset($_POST['sitemap_url']) || empty($_POST['sitemap_url'])) {
834 + //error_log('[MXCHAT-URL] Error: Empty or missing URL');
1414 835 set_transient('mxchat_admin_notice_error',
1415 836 esc_html__('Please provide a valid URL.', 'mxchat'),
1416 837 30
1417 838 );
@@ -1419,56 +840,43 @@
1419 840 exit;
1420 841 }
1421 842
1422 843 $submitted_url = esc_url_raw($_POST['sitemap_url']);
844 + //error_log('[MXCHAT-URL] Processing URL: ' . $submitted_url);
1423 845
1424 - // Convert Google Drive sharing URLs to direct download URLs
1425 - if ( strpos($submitted_url, 'drive.google.com') !== false ) {
1426 - $file_id = '';
1427 - if ( preg_match('/[?&]id=([a-zA-Z0-9_-]+)/', $submitted_url, $m) ) {
1428 - $file_id = $m[1];
1429 - } elseif ( preg_match('#/file/d/([a-zA-Z0-9_-]+)#', $submitted_url, $m) ) {
1430 - $file_id = $m[1];
1431 - }
1432 - if ( ! empty($file_id) ) {
1433 - $submitted_url = 'https://drive.google.com/uc?export=download&id=' . $file_id;
1434 - }
846 + // Validate API key first
847 + $options = get_option('mxchat_options');
848 + $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
849 +
850 + if (strpos($selected_model, 'voyage') === 0) {
851 + $api_key = $options['voyage_api_key'] ?? '';
852 + $provider_name = 'Voyage AI';
853 + } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
854 + $api_key = $options['gemini_api_key'] ?? '';
855 + $provider_name = 'Google Gemini';
856 + } else {
857 + $api_key = $options['api_key'] ?? '';
858 + $provider_name = 'OpenAI';
1435 859 }
1436 -
1437 - // Get bot_id from form submission
1438 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1439 -
1440 - // Get bot-specific options and validate the embedding decision —
1441 - // custom-provider-aware (plan cbd5fd).
1442 - $bot_options = $this->get_bot_options($bot_id);
1443 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
1444 -
1445 - $preflight = MxChat_Utils::embedding_preflight($options);
1446 - if (!$preflight['ok']) {
1447 - set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
860 +
861 + if (empty($api_key)) {
862 + $error_message = sprintf(
863 + esc_html__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
864 + $provider_name
865 + );
866 + //error_log('[MXCHAT-URL] Error: ' . $error_message);
867 + set_transient('mxchat_admin_notice_error', $error_message, 30);
868 + //error_log('[MXCHAT-URL] Set error transient: ' . $error_message);
1448 869 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1449 870 exit;
1450 871 }
1451 - $api_key = $preflight['api_key'];
1452 872
1453 - // Fetch URL — send an honest, versioned MXChat crawler UA (not a spoofed
1454 - // browser). Stale browser UAs are exactly what WAFs like SiteGround's
1455 - // ModSecurity flag as scrapers, 403-ing the fetch (including PDFs served
1456 - // from the site's own media library, which route through this same call).
1457 - // See mxchat_ingest_user_agent(). Accept is kept for content negotiation;
1458 - // the browser-only Accept-Language fingerprint is dropped so it stays
1459 - // coherent with a bot identity.
1460 - $response = wp_remote_get($submitted_url, array(
1461 - 'timeout' => 30,
1462 - 'sslverify' => false,
1463 - 'user-agent' => mxchat_ingest_user_agent(),
1464 - 'headers' => array(
1465 - 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1466 - ),
1467 - ));
873 + //error_log('[MXCHAT-URL] Fetching URL content');
874 + $response = wp_remote_get($submitted_url, array('timeout' => 30));
1468 875
1469 876 if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
1470 877 $error_message = is_wp_error($response) ? $response->get_error_message() : 'HTTP Status: ' . wp_remote_retrieve_response_code($response);
878 + //error_log('[MXCHAT-URL] Error fetching URL: ' . $error_message);
1471 879 set_transient('mxchat_admin_notice_error',
1472 880 sprintf(
1473 881 esc_html__('Failed to fetch the URL: %s', 'mxchat'),
1474 882 esc_html($error_message)
@@ -1479,11 +887,13 @@
1479 887 exit;
1480 888 }
1481 889
1482 890 $content_type = wp_remote_retrieve_header($response, 'content-type');
891 + //error_log('[MXCHAT-URL] Content type: ' . $content_type);
1483 892 $body_content = wp_remote_retrieve_body($response);
1484 893
1485 894 if (empty($body_content)) {
895 + //error_log('[MXCHAT-URL] Error: Empty response body');
1486 896 set_transient('mxchat_admin_notice_error',
1487 897 esc_html__('Empty response received from URL.', 'mxchat'),
1488 898 30
1489 899 );
@@ -1489,21 +899,29 @@
1489 899 );
1490 900 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1491 901 exit;
1492 902 }
903 + //error_log('[MXCHAT-URL] Retrieved body content length: ' . strlen($body_content) . ' bytes');
1493 904
1494 905 // Handle PDF URL
1495 906 if ($this->mxchat_is_pdf_url($submitted_url, $response)) {
1496 - $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response, $bot_id);
907 + //error_log('[MXCHAT-URL] Detected PDF URL, handling PDF for knowledge base');
908 + $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response);
909 + //error_log('[MXCHAT-URL] PDF handling result: ' . $result);
1497 910
1498 - if ($result === 'queued') {
1499 - set_transient('mxchat_admin_notice_success',
1500 - esc_html__('PDF queued for processing. Processing will start automatically.', 'mxchat'),
911 + if ($result === 'scheduled') {
912 + set_transient(
913 + 'mxchat_last_pdf_url',
914 + sanitize_text_field($submitted_url),
915 + DAY_IN_SECONDS
916 + );
917 + set_transient('mxchat_admin_notice_info',
918 + esc_html__('PDF processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1501 919 30
1502 920 );
1503 921 } else {
1504 922 set_transient('mxchat_admin_notice_error',
1505 - esc_html__('Failed to queue PDF processing: ', 'mxchat') . esc_html($result),
923 + esc_html__('Failed to start PDF processing: ', 'mxchat') . esc_html($result),
1506 924 30
1507 925 );
1508 926 }
1509 927
@@ -1512,8 +930,9 @@
1512 930 }
1513 931
1514 932 // Handle Sitemap XML
1515 933 if (strpos($content_type, 'xml') !== false || strpos($body_content, '<urlset') !== false) {
934 + //error_log('[MXCHAT-URL] Detected XML content, processing as sitemap');
1516 935 libxml_use_internal_errors(true);
1517 936 $xml = simplexml_load_string($body_content);
1518 937 $xml_errors = libxml_get_errors();
1519 938 libxml_clear_errors();
@@ -1518,8 +937,15 @@
1518 937 $xml_errors = libxml_get_errors();
1519 938 libxml_clear_errors();
1520 939
1521 940 if ($xml === false || !empty($xml_errors)) {
941 + //error_log('[MXCHAT-URL] Error: Invalid XML format');
942 + if (!empty($xml_errors)) {
943 + foreach ($xml_errors as $error) {
944 + //error_log('[MXCHAT-URL] XML Error: ' . $error->message);
945 + }
946 + }
947 +
1522 948 set_transient('mxchat_admin_notice_error',
1523 949 esc_html__('Invalid sitemap XML. Please provide a valid sitemap.', 'mxchat'),
1524 950 30
1525 951 );
@@ -1526,30 +952,29 @@
1526 952 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1527 953 exit;
1528 954 }
1529 955
1530 - $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url, $bot_id);
956 + //error_log('[MXCHAT-URL] Valid XML found, handling sitemap for knowledge base');
957 + $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url);
958 + //error_log('[MXCHAT-URL] Sitemap handling result: ' . $result);
1531 959
1532 - if ($result === 'queued') {
1533 - set_transient('mxchat_admin_notice_success',
1534 - esc_html__('Sitemap queued for processing. Processing will start automatically.', 'mxchat'),
960 + if ($result === 'scheduled') {
961 + set_transient(
962 + 'mxchat_last_sitemap_url',
963 + sanitize_text_field($submitted_url),
964 + DAY_IN_SECONDS
965 + );
966 + set_transient('mxchat_admin_notice_info',
967 + esc_html__('Sitemap processing has started in the background. You can check the progress in the Knowledge Base section.', 'mxchat'),
1535 968 30
1536 969 );
1537 970 } else {
1538 - // Surface the reason the handler already computed (embedding pre-flight,
1539 - // empty sitemap, queue failure). The old message pointed at the status
1540 - // area, which is empty on this path — nothing was ever queued.
1541 - if (is_string($result) && $result !== '') {
1542 - set_transient('mxchat_admin_notice_error',
1543 - esc_html__('Failed to queue sitemap processing: ', 'mxchat') . esc_html($result),
1544 - 30
1545 - );
1546 - } else {
1547 - set_transient('mxchat_admin_notice_error',
1548 - esc_html__('Failed to queue sitemap processing.', 'mxchat'),
1549 - 30
1550 - );
1551 - }
971 + // Return to the admin page without a redirect for better error display
972 + // The error is already stored in the sitemap status transient
973 + set_transient('mxchat_admin_notice_error',
974 + esc_html__('Failed to start sitemap processing. Please check the status below for details.', 'mxchat'),
975 + 30
976 + );
1552 977 }
1553 978
1554 979 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1555 980 exit;
@@ -1554,50 +979,127 @@
1554 979 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1555 980 exit;
1556 981 }
1557 982
1558 - // Handle Regular URL (single page)
983 + // Handle Regular URL
984 + //error_log('[MXCHAT-URL] Processing as regular webpage');
1559 985 $page_content = $this->mxchat_extract_main_content($body_content);
986 + //error_log('[MXCHAT-URL] Extracted content length: ' . strlen($page_content) . ' bytes');
987 +
1560 988 $sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
989 + //error_log('[MXCHAT-URL] Sanitized content length: ' . strlen($sanitized_content) . ' bytes');
1561 990
1562 - //error_log('[MXCHAT-URL-DEBUG] URL: ' . $submitted_url);
1563 - //error_log('[MXCHAT-URL-DEBUG] Extracted page_content length: ' . strlen($page_content));
1564 - //error_log('[MXCHAT-URL-DEBUG] Sanitized content length: ' . strlen($sanitized_content));
1565 - //error_log('[MXCHAT-URL-DEBUG] Content preview: ' . substr($sanitized_content, 0, 500));
1566 -
1567 991 if (empty($sanitized_content)) {
992 + //error_log('[MXCHAT-URL] Error: No valid content after sanitization');
993 +
994 + // Set both transients - the error notice and the URL status
1568 995 set_transient('mxchat_admin_notice_error',
1569 996 esc_html__('No valid content found on the provided URL.', 'mxchat'),
1570 997 30
1571 998 );
999 +
1000 + // Set URL status transient
1001 + set_transient('mxchat_single_url_status', [
1002 + 'url' => $submitted_url,
1003 + 'timestamp' => current_time('mysql'),
1004 + 'status' => 'failed',
1005 + 'error' => esc_html__('No valid content found on the provided URL.', 'mxchat')
1006 + ], DAY_IN_SECONDS);
1007 +
1572 1008 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1573 1009 exit;
1574 1010 }
1575 1011
1576 - // For single URLs, process immediately using submit_content_to_db
1577 - // This handles chunking automatically for large content
1578 - $db_result = MxChat_Utils::submit_content_to_db(
1579 - $sanitized_content,
1580 - $submitted_url,
1581 - $api_key,
1582 - null,
1583 - $bot_id,
1584 - 'url' // content_type
1585 - );
1012 + //error_log('[MXCHAT-URL] Generating embedding for content');
1013 + $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
1586 1014
1587 - if (is_wp_error($db_result)) {
1588 - $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
1015 + // Check if embedding_vector is a string (error message)
1016 + if (is_string($embedding_vector)) {
1017 + //error_log('[MXCHAT-URL] Error generating embedding: ' . $embedding_vector);
1018 + $error_message = esc_html__('Failed to generate embedding: ', 'mxchat') . esc_html($embedding_vector);
1019 + //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
1020 +
1021 + // Set both transients
1589 1022 set_transient('mxchat_admin_notice_error', $error_message, 30);
1590 - } else {
1023 +
1024 + // Set URL status transient
1025 + set_transient('mxchat_single_url_status', [
1026 + 'url' => $submitted_url,
1027 + 'timestamp' => current_time('mysql'),
1028 + 'status' => 'failed',
1029 + 'error' => $error_message
1030 + ], DAY_IN_SECONDS);
1031 +
1032 + wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1033 + exit;
1034 + }
1035 +
1036 + if (is_array($embedding_vector)) {
1037 + //error_log('[MXCHAT-URL] Successfully generated embedding with ' . count($embedding_vector) . ' dimensions');
1038 +
1039 + $db_result = MxChat_Utils::submit_content_to_db(
1040 + $sanitized_content,
1041 + $submitted_url,
1042 + $api_key
1043 + );
1044 +
1045 + if (is_wp_error($db_result)) {
1046 + //error_log('[MXCHAT-URL] Error: Failed to store content in database: ' . $db_result->get_error_message());
1047 + $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
1048 + //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
1049 +
1050 + // Set both transients
1051 + set_transient('mxchat_admin_notice_error', $error_message, 30);
1052 +
1053 + // Set URL status transient
1054 + set_transient('mxchat_single_url_status', [
1055 + 'url' => $submitted_url,
1056 + 'timestamp' => current_time('mysql'),
1057 + 'status' => 'failed',
1058 + 'error' => $error_message
1059 + ], DAY_IN_SECONDS);
1060 +
1061 + wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1062 + exit;
1063 + }
1064 +
1065 + //error_log('[MXCHAT-URL] Successfully stored content in database');
1591 1066 $success_message = esc_html__('URL content successfully submitted!', 'mxchat');
1067 + //error_log('[MXCHAT-URL] Setting success transient: ' . $success_message);
1068 +
1069 + // Set both transients
1592 1070 set_transient('mxchat_admin_notice_success', $success_message, 30);
1071 +
1072 + // Set URL status transient with success
1073 + set_transient('mxchat_single_url_status', [
1074 + 'url' => $submitted_url,
1075 + 'timestamp' => current_time('mysql'),
1076 + 'status' => 'complete',
1077 + 'content_length' => strlen($sanitized_content),
1078 + 'embedding_dimensions' => count($embedding_vector)
1079 + ], DAY_IN_SECONDS);
1080 +
1081 + } else {
1082 + //error_log('[MXCHAT-URL] Error: Failed to generate embedding. Unexpected result type: ' . gettype($embedding_vector));
1083 + $error_message = esc_html__('Failed to generate embedding: Unexpected result type. Please check your API key and try again.', 'mxchat');
1084 + //error_log('[MXCHAT-URL] Setting error transient: ' . $error_message);
1085 +
1086 + // Set both transients
1087 + set_transient('mxchat_admin_notice_error', $error_message, 30);
1088 +
1089 + // Set URL status transient
1090 + set_transient('mxchat_single_url_status', [
1091 + 'url' => $submitted_url,
1092 + 'timestamp' => current_time('mysql'),
1093 + 'status' => 'failed',
1094 + 'error' => $error_message
1095 + ], DAY_IN_SECONDS);
1593 1096 }
1594 1097
1098 + //error_log('[MXCHAT-URL] ===== Completed URL submission process =====');
1595 1099 wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
1596 1100 exit;
1597 1101 }
1598 -
1599 -
1600 1102 public function mxchat_get_single_url_status() {
1601 1103 $status = get_transient('mxchat_single_url_status');
1602 1104 if (!$status) {
1603 1105 return null;
@@ -1609,11 +1111,13 @@
1609 1111 }
1610 1112
1611 1113 return $status;
1612 1114 }
1613 -
1614 -public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url, $bot_id = 'default') {
1115 +public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url) {
1116 + // Clear any single URL status when starting sitemap processing
1117 + delete_transient('mxchat_single_url_status');
1615 1118 if (!current_user_can('manage_options')) {
1119 + //error_log(esc_html__('Unauthorized sitemap processing attempt', 'mxchat'));
1616 1120 return false;
1617 1121 }
1618 1122
1619 1123 try {
@@ -1622,96 +1126,331 @@
1622 1126 if (!$xml || !is_object($xml)) {
1623 1127 throw new Exception(__('Invalid XML object provided', 'mxchat'));
1624 1128 }
1625 1129
1626 - // Get bot-specific embedding API for validation
1627 - $bot_options = $this->get_bot_options($bot_id);
1628 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
1130 + // Add embedding validation before processing
1131 + // Test embedding with a small sample text to verify API key is working
1132 + $test_result = $this->mxchat_generate_embedding("This is a test to verify the embedding API key is working.");
1629 1133
1630 - // Test the embedding API before processing
1631 - $test_phrase = "Test embedding generation for MxChat";
1632 - $test_result = $this->mxchat_generate_embedding($test_phrase, $bot_id);
1633 -
1134 + // Check if test_result is a string (error message) rather than an array (valid embedding)
1634 1135 if (is_string($test_result)) {
1136 + //error_log('[MXCHAT-URL] Embedding API validation failed: ' . $test_result);
1137 +
1138 + // Store the error in the status transient so it can be displayed later
1139 + $status_data = array(
1140 + 'total_urls' => 0,
1141 + 'processed_urls' => 0,
1142 + 'status' => 'error',
1143 + 'error' => __('Embedding API validation failed: ', 'mxchat') . $test_result,
1144 + 'last_update' => time()
1145 + );
1146 +
1147 + set_transient(
1148 + sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
1149 + array_map('sanitize_text_field', $status_data),
1150 + DAY_IN_SECONDS
1151 + );
1152 +
1635 1153 throw new Exception(__('Embedding API validation failed: ', 'mxchat') . $test_result);
1636 1154 }
1637 1155
1156 + // Make sure it's an array (valid embedding)
1638 1157 if (!is_array($test_result)) {
1158 + //error_log('[MXCHAT-URL] Embedding API returned unexpected result type: ' . gettype($test_result));
1639 1159 throw new Exception(__('Embedding API returned unexpected result type. Please check your configuration.', 'mxchat'));
1640 1160 }
1641 1161
1642 - // Extract URLs from sitemap
1643 - $urls = array();
1162 + $urls = [];
1644 1163 foreach ($xml->url as $url_element) {
1645 1164 $url = esc_url_raw((string)$url_element->loc);
1646 1165 if ($url) {
1647 - $urls[] = array('url' => $url);
1166 + $urls[] = $url;
1648 1167 }
1649 1168 }
1650 1169
1651 - $total_urls = count($urls);
1170 + $total_urls = absint(count($urls));
1652 1171
1653 1172 if ($total_urls < 1) {
1654 1173 throw new Exception(__('No valid URLs found in sitemap', 'mxchat'));
1655 1174 }
1656 1175
1657 - // Create unique queue ID
1658 - $queue_id = 'sitemap_' . md5($sitemap_url . time());
1176 + wp_schedule_single_event(time(), 'mxchat_process_sitemap_urls', array(
1177 + 'urls' => $urls,
1178 + 'sitemap_url' => $sitemap_url,
1179 + 'total_urls' => $total_urls,
1180 + 'batch_size' => absint(10),
1181 + 'batch_pause' => absint(5)
1182 + ));
1659 1183
1660 - // Add URLs to queue
1661 - $queued_count = $this->mxchat_add_to_queue($queue_id, 'url', $urls, $bot_id);
1184 + $status_data = array(
1185 + 'total_urls' => $total_urls,
1186 + 'processed_urls' => 0,
1187 + 'status' => 'processing',
1188 + 'last_update' => time()
1189 + );
1662 1190
1663 - if ($queued_count === 0) {
1664 - throw new Exception(__('Failed to add URLs to processing queue', 'mxchat'));
1665 - }
1191 + set_transient(
1192 + sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
1193 + array_map('sanitize_text_field', $status_data),
1194 + DAY_IN_SECONDS
1195 + );
1666 1196
1667 - // Store queue metadata
1668 - $this->mxchat_set_queue_meta($queue_id, 'source_url', $sitemap_url);
1669 - $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'sitemap');
1670 - $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_urls);
1671 - $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
1672 - $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));
1197 + return __('scheduled', 'mxchat');
1673 1198
1674 - // Store queue ID in transient for status tracking
1675 - set_transient('mxchat_active_queue_sitemap', $queue_id, DAY_IN_SECONDS);
1676 - set_transient('mxchat_last_sitemap_url', $sitemap_url, DAY_IN_SECONDS);
1677 -
1678 - return 'queued';
1679 -
1680 - } catch (Exception $e) {
1199 + } catch (\Exception $e) {
1681 1200 $error_message = $e->getMessage();
1682 1201 //error_log(sprintf(esc_html__('Error preparing sitemap for processing: %s', 'mxchat'), esc_html($error_message)));
1683 1202
1203 + // Store the sitemap URL and error in transients so they can be displayed
1204 + set_transient(
1205 + 'mxchat_last_sitemap_url',
1206 + sanitize_text_field($sitemap_url),
1207 + DAY_IN_SECONDS
1208 + );
1209 +
1210 + $status_data = array(
1211 + 'total_urls' => 0,
1212 + 'processed_urls' => 0,
1213 + 'status' => 'error',
1214 + 'error' => $error_message,
1215 + 'last_update' => time()
1216 + );
1217 +
1218 + set_transient(
1219 + sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url)),
1220 + array_map('sanitize_text_field', $status_data),
1221 + DAY_IN_SECONDS
1222 + );
1223 +
1684 1224 return $error_message;
1685 1225 }
1686 -
1687 1226 }
1688 1227
1689 -/**
1690 - * Remove shortcode tags but preserve the content inside them
1691 - * Example: [vc_column]Hello World[/vc_column] becomes "Hello World"
1692 - *
1693 - * @param string $content The content containing shortcodes
1694 - * @return string Content with shortcode tags removed but inner content preserved
1695 - */
1696 -private function strip_shortcode_tags_preserve_content($content) {
1697 - // Single-pass regex removes all shortcode brackets: [tag], [tag attr="val"], [/tag], [tag /]
1698 - // Content between tags is inherently preserved since only brackets are targeted
1699 - $result = preg_replace('/\[\/?\w[\w-]*[^\]]*\]/', '', $content);
1700 - return ($result !== null) ? $result : $content;
1228 +public function mxchat_process_sitemap_urls_cron($urls, $sitemap_url, $total_urls, $batch_size, $batch_pause) {
1229 + // Validate inputs
1230 + $sitemap_url = esc_url_raw($sitemap_url);
1231 + $total_urls = absint($total_urls);
1232 + $batch_size = absint($batch_size);
1233 + $batch_pause = absint($batch_pause);
1234 +
1235 + if (!is_array($urls) || empty($urls)) {
1236 + return;
1237 + }
1238 +
1239 + try {
1240 + $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
1241 + $status = get_transient($status_key);
1242 +
1243 + if (!$status || !is_array($status)) {
1244 + throw new Exception('Invalid status data retrieved from transient');
1245 + }
1246 +
1247 + // Initialize failed_urls array if it doesn't exist
1248 + if (!isset($status['failed_urls_list']) || !is_array($status['failed_urls_list'])) {
1249 + $status['failed_urls_list'] = [];
1250 + }
1251 +
1252 + $start_url = absint($status['processed_urls']);
1253 + $end_url = min($start_url + $batch_size, $total_urls);
1254 +
1255 + // Track batch statistics
1256 + $batch_stats = [
1257 + 'processed' => 0,
1258 + 'failed' => 0,
1259 + 'last_error' => '',
1260 + 'embedding_errors' => 0
1261 + ];
1262 +
1263 + // Check embedding configuration with first URL (only on first batch)
1264 + if ($start_url === 0) {
1265 + $test_url = esc_url_raw($urls[0]);
1266 + $test_response = wp_remote_get($test_url);
1267 +
1268 + if (!is_wp_error($test_response) && wp_remote_retrieve_response_code($test_response) === 200) {
1269 + $test_html = wp_remote_retrieve_body($test_response);
1270 + $test_content = $this->mxchat_extract_main_content($test_html);
1271 + $test_sanitized = $this->mxchat_sanitize_content_for_api($test_content);
1272 +
1273 + if (!empty($test_sanitized)) {
1274 + $test_embedding = $this->mxchat_generate_embedding($test_sanitized);
1275 +
1276 + if (is_string($test_embedding)) {
1277 + throw new Exception('Embedding generation failed: ' . $test_embedding);
1278 + }
1279 +
1280 + if (!is_array($test_embedding)) {
1281 + throw new Exception('Embedding generation returned unexpected result type: ' . gettype($test_embedding));
1282 + }
1283 + }
1284 + }
1285 + }
1286 +
1287 + for ($i = $start_url; $i < $end_url; $i++) {
1288 + $page_url = esc_url_raw($urls[$i]);
1289 + $max_retries = 3;
1290 + $retry_count = 0;
1291 + $url_processed = false;
1292 + $last_error = '';
1293 +
1294 + while (!$url_processed && $retry_count < $max_retries) {
1295 + try {
1296 + // Attempt to fetch the URL
1297 + $page_response = wp_remote_get($page_url, array('timeout' => 30));
1298 +
1299 + if (is_wp_error($page_response)) {
1300 + throw new Exception('HTTP request failed: ' . $page_response->get_error_message());
1301 + }
1302 +
1303 + $response_code = wp_remote_retrieve_response_code($page_response);
1304 + if ($response_code !== 200) {
1305 + throw new Exception('HTTP Status: ' . $response_code);
1306 + }
1307 +
1308 + $page_html = wp_remote_retrieve_body($page_response);
1309 +
1310 + if (empty($page_html)) {
1311 + throw new Exception('Empty response body');
1312 + }
1313 +
1314 + $page_content = $this->mxchat_extract_main_content($page_html);
1315 + $sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);
1316 +
1317 + if (empty($sanitized_content)) {
1318 + throw new Exception('No valid content found after processing');
1319 + }
1320 +
1321 + $embedding_vector = $this->mxchat_generate_embedding($sanitized_content);
1322 +
1323 + if (is_string($embedding_vector)) {
1324 + throw new Exception('Embedding generation failed: ' . $embedding_vector);
1325 + }
1326 +
1327 + if (!is_array($embedding_vector)) {
1328 + throw new Exception('Embedding generation returned unexpected result type: ' . gettype($embedding_vector));
1329 + }
1330 +
1331 + // Submit to database
1332 + $options = get_option('mxchat_options');
1333 + $submission_result = MxChat_Utils::submit_content_to_db($sanitized_content, $page_url, $options['api_key']);
1334 +
1335 + if (is_wp_error($submission_result)) {
1336 + throw new Exception('Database submission failed: ' . $submission_result->get_error_message());
1337 + }
1338 +
1339 + // Success!
1340 + $url_processed = true;
1341 + $batch_stats['processed']++;
1342 +
1343 + } catch (Exception $e) {
1344 + $retry_count++;
1345 + $last_error = $e->getMessage();
1346 +
1347 + //error_log("URL {$page_url} failed (attempt {$retry_count}/{$max_retries}): " . $last_error);
1348 +
1349 + // Track embedding errors specifically
1350 + if (strpos($last_error, 'Embedding') !== false) {
1351 + $batch_stats['embedding_errors']++;
1352 + }
1353 +
1354 + if ($retry_count < $max_retries) {
1355 + // Wait before retry (exponential backoff: 1s, 2s, 4s)
1356 + sleep(pow(2, $retry_count - 1));
1357 + }
1358 + }
1359 + }
1360 +
1361 + // If URL still not processed after all retries, mark as failed
1362 + if (!$url_processed) {
1363 + $batch_stats['failed']++;
1364 + $batch_stats['last_error'] = $last_error;
1365 +
1366 + // Add to failed URLs list
1367 + $status['failed_urls_list'][] = [
1368 + 'url' => $page_url,
1369 + 'error' => $last_error,
1370 + 'time' => time(),
1371 + 'retries' => $max_retries
1372 + ];
1373 +
1374 + // Limit the number of failed URLs we store to prevent transient size issues
1375 + if (count($status['failed_urls_list']) > 100) {
1376 + $status['failed_urls_list'] = array_slice($status['failed_urls_list'], -100);
1377 + }
1378 + }
1379 +
1380 + // Update progress
1381 + $status['processed_urls'] = absint($i + 1);
1382 + $status['last_update'] = time();
1383 + $status['failed_urls'] = absint($status['failed_urls'] ?? 0) + ($url_processed ? 0 : 1);
1384 + $status['last_error'] = $batch_stats['last_error'];
1385 +
1386 + set_transient($status_key, $status, DAY_IN_SECONDS);
1387 +
1388 + // If we have too many consecutive embedding errors, stop processing
1389 + if ($batch_stats['embedding_errors'] >= 10) {
1390 + throw new Exception('Too many consecutive embedding failures detected. Please check your API configuration.');
1391 + }
1392 + }
1393 +
1394 + // If all URLs in this batch failed, stop processing
1395 + if ($batch_stats['processed'] === 0 && $batch_stats['failed'] > 0) {
1396 + $status['status'] = 'error';
1397 + $status['error'] = sprintf(
1398 + 'Processing stopped: %d consecutive failures in batch. Last error: %s',
1399 + $batch_stats['failed'],
1400 + $batch_stats['last_error']
1401 + );
1402 + set_transient($status_key, $status, DAY_IN_SECONDS);
1403 + return;
1404 + }
1405 +
1406 + // Update final progress
1407 + $status['processed_urls'] = min($end_url, $total_urls);
1408 + $status['last_update'] = time();
1409 + set_transient($status_key, $status, DAY_IN_SECONDS);
1410 +
1411 + // Check if we've processed all URLs
1412 + if ($end_url >= $total_urls) {
1413 + // All URLs have been processed - mark as complete
1414 + $status['status'] = 'complete';
1415 + $status['processed_urls'] = $total_urls;
1416 +
1417 + // Add completion summary
1418 + $status['completion_summary'] = [
1419 + 'total_urls' => $total_urls,
1420 + 'successful_urls' => $total_urls - absint($status['failed_urls'] ?? 0),
1421 + 'failed_urls' => absint($status['failed_urls'] ?? 0),
1422 + 'completion_time' => current_time('mysql')
1423 + ];
1424 +
1425 + // Save the completed status (don't delete it - let user dismiss manually)
1426 + set_transient($status_key, $status, DAY_IN_SECONDS);
1427 +
1428 + // DON'T delete the status transients here - let user dismiss manually
1429 + } else {
1430 + // Schedule next batch
1431 + wp_schedule_single_event(time() + $batch_pause, 'mxchat_process_sitemap_urls', array(
1432 + 'urls' => $urls,
1433 + 'sitemap_url' => $sitemap_url,
1434 + 'total_urls' => $total_urls,
1435 + 'batch_size' => $batch_size,
1436 + 'batch_pause' => $batch_pause,
1437 + ));
1438 + }
1439 + } catch (\Exception $e) {
1440 + $status['status'] = 'error';
1441 + $status['error'] = $e->getMessage();
1442 + set_transient($status_key, $status, DAY_IN_SECONDS);
1443 + }
1701 1444 }
1702 -
1703 1445 public function mxchat_sanitize_content_for_api($content) {
1704 1446 //error_log('[MXCHAT-SANITIZE] Original content preview: ' . substr($content, 0, 500) . '...');
1705 -
1706 - // Remove shortcode tags but PRESERVE content inside them
1707 - $content = $this->strip_shortcode_tags_preserve_content($content);
1708 -
1447 +
1709 1448 // Remove script, style tags, and HTML comments
1710 1449 $content = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $content);
1711 1450 $content = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $content);
1712 1451 $content = preg_replace('/<!--(.|\s)*?-->/', '', $content);
1713 -
1452 +
1714 1453 // Remove all HTML tags and decode HTML entities
1715 1454 $content = wp_strip_all_tags($content);
1716 1455 $content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5);
1717 1456
@@ -1737,20 +1476,14 @@
1737 1476
1738 1477 // Ensure valid UTF-8 encoding
1739 1478 $content = wp_check_invalid_utf8($content);
1740 1479
1741 - // Remove extremely long runs with no whitespace (base64 blobs, minified JS).
1742 - // Counts CHARACTERS (/u), and never strips a run containing characters from a
1743 - // script written without spaces — Japanese, Chinese, Thai, Khmer, Lao, Myanmar —
1744 - // where a normal paragraph is legitimately one unbroken run.
1745 - $content = preg_replace_callback('/\S{300,}/u', function ($m) {
1746 - return preg_match('/[\p{Han}\p{Hiragana}\p{Katakana}\p{Thai}\p{Khmer}\p{Lao}\p{Myanmar}]/u', $m[0]) ? $m[0] : ' ';
1747 - }, $content);
1748 -
1749 - // Remove emoji/symbol blocks only — not the whole supplementary plane, which
1750 - // also holds CJK Extension B ideographs used in real Chinese/Japanese names
1751 - $content = preg_replace('/[\x{1F000}-\x{1F0FF}\x{1F300}-\x{1FAFF}]/u', '', $content);
1480 + // Remove any extremely long strings without spaces (often garbage)
1481 + $content = preg_replace('/\S{300,}/', ' ', $content);
1752 1482
1483 + // Replace problematic characters that often cause database issues
1484 + $content = preg_replace('/[\x{10000}-\x{10FFFF}]/u', '', $content); // Remove emoji and other high Unicode characters
1485 +
1753 1486 // Replace any remaining potentially problematic characters with spaces
1754 1487 // BUT preserve newlines by temporarily replacing them
1755 1488 $content = str_replace("\n", "NEWLINE_PLACEHOLDER", $content);
1756 1489 $content = preg_replace('/[^\p{L}\p{N}\p{P}\p{Z}\p{Sm}]/u', ' ', $content);
@@ -1755,13 +1488,12 @@
1755 1488 $content = str_replace("\n", "NEWLINE_PLACEHOLDER", $content);
1756 1489 $content = preg_replace('/[^\p{L}\p{N}\p{P}\p{Z}\p{Sm}]/u', ' ', $content);
1757 1490 $content = str_replace("NEWLINE_PLACEHOLDER", "\n", $content);
1758 1491
1759 - // Limit to reasonable length if needed (byte limit — MySQL TEXT is byte-sized,
1760 - // but cut on a character boundary so a multibyte char is never split mid-sequence)
1492 + // Limit to reasonable length if needed
1761 1493 $max_length = 65000; // Just under MySQL TEXT field limit
1762 1494 if (strlen($content) > $max_length) {
1763 - $content = mb_strcut($content, 0, $max_length, 'UTF-8');
1495 + $content = substr($content, 0, $max_length);
1764 1496 }
1765 1497
1766 1498 //error_log('[MXCHAT-SANITIZE] Sanitized content preview: ' . substr($content, 0, 500) . '...');
1767 1499 return $content;
@@ -1776,12 +1508,12 @@
1776 1508 @$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'), LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
1777 1509 $xpath = new DOMXPath($dom);
1778 1510
1779 1511 // For debugging purposes
1780 - $debugEnabled = true; // Set to true to enable debugging output
1512 + $debugEnabled = false; // Set to true to enable debugging output
1781 1513 $debug = function($message) use ($debugEnabled) {
1782 1514 if ($debugEnabled) {
1783 - //error_log('[MXCHAT-EXTRACT-DEBUG] ' . $message);
1515 + //error_log('[MXCHAT-DEBUG] ' . $message);
1784 1516 }
1785 1517 };
1786 1518
1787 1519 // Direct targeting for Gerow theme posts
@@ -1880,25 +1612,18 @@
1880 1612 '//main',
1881 1613 '//div[contains(@class, "content")]'
1882 1614 ];
1883 1615
1884 - // First handle Elementor content - get only leaf widget containers to avoid duplicates
1616 + // First handle Elementor content
1885 1617 $debug("Checking for Elementor content");
1886 - // Get widget containers that are direct children of widgets (not nested inside other widget containers)
1887 - $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-widget")]//div[contains(@class, "elementor-widget-container")]');
1618 + $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-element")]//div[contains(@class, "elementor-widget-container")]');
1888 1619 if ($elementor_widgets && $elementor_widgets->length > 0) {
1889 1620 $debug("Found Elementor widgets");
1890 - $seen_content = array(); // Track seen content to avoid duplicates
1891 1621 $combined_content = '';
1892 1622 foreach ($elementor_widgets as $widget) {
1893 1623 $widget_content = $dom->saveHTML($widget);
1894 1624 if (!empty($widget_content)) {
1895 - // Create a hash of the content to detect duplicates
1896 - $content_hash = md5($widget_content);
1897 - if (!isset($seen_content[$content_hash])) {
1898 - $seen_content[$content_hash] = true;
1899 - $combined_content .= $widget_content;
1900 - }
1625 + $combined_content .= $widget_content;
1901 1626 }
1902 1627 }
1903 1628 if (!empty($combined_content)) {
1904 1629 $debug("Returning Elementor content");
@@ -1910,14 +1635,15 @@
1910 1635 foreach ($selectors as $selector) {
1911 1636 $debug("Trying selector: " . $selector);
1912 1637 $nodes = $xpath->query($selector);
1913 1638 if ($nodes && $nodes->length > 0) {
1914 - $debug("Found " . $nodes->length . " matches for selector: " . $selector);
1915 - // Only take the FIRST matching node to avoid duplicate content
1916 - // (pages often have nested or multiple containers with same class)
1917 - $content = $dom->saveHTML($nodes->item(0));
1639 + $debug("Found matches for selector: " . $selector);
1640 + $content = '';
1641 + foreach ($nodes as $node) {
1642 + $content .= $dom->saveHTML($node);
1643 + }
1918 1644 if (!empty($content)) {
1919 - $debug("Returning content from selector: " . $selector . " (first match only)");
1645 + $debug("Returning content from selector: " . $selector);
1920 1646 return $content;
1921 1647 }
1922 1648 }
1923 1649 }
@@ -1941,106 +1667,16 @@
1941 1667 $debug("Returning blog-area section content");
1942 1668 return $content;
1943 1669 }
1944 1670 }
1945 -
1946 - // Generic container selectors for non-CMS sites (like .asp pages)
1947 - $debug("Trying generic container selectors");
1948 - $generic_selectors = [
1949 - '//div[@id="main"]',
1950 - '//div[@id="wrapper"]',
1951 - '//div[@id="page"]',
1952 - '//div[@id="site-content"]',
1953 - '//div[contains(@class, "main-content")]',
1954 - '//div[contains(@class, "page-content")]',
1955 - '//div[contains(@class, "site-content")]',
1956 - ];
1957 -
1958 - foreach ($generic_selectors as $selector) {
1959 - $debug("Trying generic selector: " . $selector);
1960 - $nodes = $xpath->query($selector);
1961 - if ($nodes && $nodes->length > 0) {
1962 - $content = $dom->saveHTML($nodes->item(0));
1963 - if (!empty($content)) {
1964 - $debug("Returning content from generic selector: " . $selector);
1965 - return $content;
1966 - }
1967 - }
1968 - }
1969 -
1970 - // Paragraph-based content detection - find regions with substantial text
1971 - $debug("Trying paragraph-based content detection");
1972 - $paragraphs = $xpath->query('//p[string-length(normalize-space()) > 50]');
1973 - if ($paragraphs && $paragraphs->length >= 3) {
1974 - $debug("Found " . $paragraphs->length . " substantial paragraphs");
1975 - // Collect all substantial paragraphs and their content
1976 - $paragraph_content = '';
1977 - foreach ($paragraphs as $p) {
1978 - $paragraph_content .= $dom->saveHTML($p) . "\n";
1979 - }
1980 - if (!empty($paragraph_content)) {
1981 - $debug("Returning paragraph-based content");
1982 - return $paragraph_content;
1983 - }
1984 - }
1985 -
1986 - // Improved body fallback - strip nav/header/footer elements first
1987 - $debug("Using improved body fallback");
1671 +
1672 + // Fallback: Return the body content if no specific selector matches
1673 + $debug("Using body fallback");
1988 1674 $body = $dom->getElementsByTagName('body');
1989 1675 if ($body->length > 0) {
1990 - // Clone the body to avoid modifying the original DOM
1991 - $body_clone = $body->item(0)->cloneNode(true);
1992 -
1993 - // Remove common non-content elements by tag name
1994 - $remove_tags = ['nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript'];
1995 - foreach ($remove_tags as $tag) {
1996 - $elements = $body_clone->getElementsByTagName($tag);
1997 - // Iterate backwards to safely remove elements
1998 - for ($i = $elements->length - 1; $i >= 0; $i--) {
1999 - $el = $elements->item($i);
2000 - if ($el && $el->parentNode) {
2001 - $el->parentNode->removeChild($el);
2002 - }
2003 - }
2004 - }
2005 -
2006 - // Remove elements with common non-content class names using XPath on the cloned body
2007 - $temp_dom = new DOMDocument();
2008 - @$temp_dom->appendChild($temp_dom->importNode($body_clone, true));
2009 - $temp_xpath = new DOMXPath($temp_dom);
2010 -
2011 - $remove_class_patterns = [
2012 - '//*[contains(@class, "nav")]',
2013 - '//*[contains(@class, "menu")]',
2014 - '//*[contains(@class, "sidebar")]',
2015 - '//*[contains(@class, "footer")]',
2016 - '//*[contains(@class, "header")]',
2017 - '//*[contains(@id, "nav")]',
2018 - '//*[contains(@id, "menu")]',
2019 - '//*[contains(@id, "sidebar")]',
2020 - '//*[contains(@id, "footer")]',
2021 - '//*[contains(@id, "header")]',
2022 - ];
2023 -
2024 - foreach ($remove_class_patterns as $pattern) {
2025 - $elements = $temp_xpath->query($pattern);
2026 - if ($elements) {
2027 - for ($i = $elements->length - 1; $i >= 0; $i--) {
2028 - $el = $elements->item($i);
2029 - if ($el && $el->parentNode) {
2030 - $el->parentNode->removeChild($el);
2031 - }
2032 - }
2033 - }
2034 - }
2035 -
2036 - $cleaned_content = $temp_dom->saveHTML();
2037 - if (!empty($cleaned_content)) {
2038 - $debug("Returning cleaned body content");
2039 - return $cleaned_content;
2040 - }
1676 + return $dom->saveHTML($body->item(0));
2041 1677 }
2042 -
1678 +
2043 1679 // Last resort: return the original HTML
2044 1680 $debug("Returning original HTML");
2045 1681 return $html;
2046 1682 } catch (Exception $e) {
@@ -2092,41 +1728,47 @@
2092 1728 try {
2093 1729 // Verify the request
2094 1730 check_ajax_referer('mxchat_status_nonce', 'nonce');
2095 1731
2096 - // Get active queue IDs
2097 - $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
2098 - $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
1732 + // Get the status just like in your admin page
1733 + $pdf_url = get_transient('mxchat_last_pdf_url');
1734 + $sitemap_url = get_transient('mxchat_last_sitemap_url');
2099 1735
2100 - $sitemap_status = false;
2101 - $pdf_status = false;
1736 + $pdf_status = $pdf_url ? $this->mxchat_get_pdf_processing_status($pdf_url) : false;
1737 + $sitemap_status = $sitemap_url ? $this->mxchat_get_sitemap_processing_status($sitemap_url) : false;
2102 1738
2103 - // Get sitemap queue status
2104 - if ($sitemap_queue_id) {
2105 - $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
1739 + // Add the PDF URL to the status object
1740 + if ($pdf_status && $pdf_url) {
1741 + $pdf_status['pdf_url'] = $pdf_url;
2106 1742 }
2107 1743
2108 - // Get PDF queue status
2109 - if ($pdf_queue_id) {
2110 - $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
2111 - }
1744 + // Set the current PDF URL for the manual batch processing button
1745 + $current_pdf_url = $pdf_url;
2112 1746
1747 + // Check for true processing status, not just presence of status
2113 1748 $is_active_processing =
2114 - ($sitemap_status && $sitemap_status['status'] === 'processing') ||
2115 - ($pdf_status && $pdf_status['status'] === 'processing');
1749 + ($sitemap_status && isset($sitemap_status['status']) && $sitemap_status['status'] === 'processing') ||
1750 + ($pdf_status && isset($pdf_status['status']) && $pdf_status['status'] === 'processing');
2116 1751
1752 + // Get single URL status, but only if no processing is active
1753 + $single_url_status = !$is_active_processing ? $this->mxchat_get_single_url_status() : false;
1754 +
1755 + // REMOVED: Auto-clearing of completed status - now only done via dismiss button
1756 +
2117 1757 // Return JSON response with the status data
2118 1758 wp_send_json(array(
2119 1759 'pdf_status' => $pdf_status,
2120 1760 'sitemap_status' => $sitemap_status,
1761 + 'single_url_status' => $single_url_status,
2121 1762 'is_processing' => $is_active_processing,
2122 - 'sitemap_queue_id' => $sitemap_queue_id,
2123 - 'pdf_queue_id' => $pdf_queue_id
1763 + 'current_pdf_url' => $current_pdf_url
2124 1764 ));
2125 1765
2126 1766 } catch (Exception $e) {
1767 + // Log the error
2127 1768 //error_log('MxChat Status Update Error: ' . $e->getMessage());
2128 1769
1770 + // Return a friendly error response
2129 1771 wp_send_json_error(array(
2130 1772 'message' => 'Error getting status updates: ' . $e->getMessage(),
2131 1773 'status' => 'error'
2132 1774 ));
@@ -2131,1371 +1773,8 @@
2131 1773 'status' => 'error'
2132 1774 ));
2133 1775 }
2134 1776 }
2135 -
2136 -/**
2137 - * Helper function to get queue status data
2138 - */
2139 -private function mxchat_get_queue_status_data($queue_id, $type = 'sitemap') {
2140 - global $wpdb;
2141 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
2142 -
2143 - // Get counts by status
2144 - $counts = $wpdb->get_results($wpdb->prepare(
2145 - "SELECT status, COUNT(*) as count
2146 - FROM $table_name
2147 - WHERE queue_id = %s
2148 - GROUP BY status",
2149 - $queue_id
2150 - ), OBJECT_K);
2151 -
2152 - $total = 0;
2153 - $completed = 0;
2154 - $failed = 0;
2155 - $processing = 0;
2156 - $pending = 0;
2157 -
2158 - foreach ($counts as $status => $data) {
2159 - $count = absint($data->count);
2160 - $total += $count;
2161 -
2162 - switch ($status) {
2163 - case 'completed':
2164 - $completed = $count;
2165 - break;
2166 - case 'failed':
2167 - $failed = $count;
2168 - break;
2169 - case 'processing':
2170 - $processing = $count;
2171 - break;
2172 - case 'pending':
2173 - $pending = $count;
2174 - break;
2175 - }
2176 - }
2177 -
2178 - if ($total === 0) {
2179 - return false;
2180 - }
2181 -
2182 - // Calculate percentage
2183 - $percentage = round((($completed + $failed) / $total) * 100);
2184 -
2185 - // Get failed items details (limit to 50)
2186 - $failed_items = array();
2187 - if ($failed > 0) {
2188 - $failed_results = $wpdb->get_results($wpdb->prepare(
2189 - "SELECT item_type, item_data, error_message, attempts, completed_at
2190 - FROM $table_name
2191 - WHERE queue_id = %s
2192 - AND status = 'failed'
2193 - AND attempts >= max_attempts
2194 - ORDER BY id DESC
2195 - LIMIT 50",
2196 - $queue_id
2197 - ));
2198 -
2199 - foreach ($failed_results as $item) {
2200 - $data = json_decode($item->item_data, true);
2201 - $url = $type === 'pdf' ? ($data['pdf_url'] ?? '') : ($data['url'] ?? '');
2202 -
2203 - $failed_items[] = array(
2204 - 'url' => $url,
2205 - 'page' => $type === 'pdf' ? ($data['page_number'] ?? 0) : 0,
2206 - 'error' => $item->error_message,
2207 - 'retries' => $item->attempts,
2208 - 'time' => strtotime($item->completed_at)
2209 - );
2210 - }
2211 - }
2212 -
2213 - // Get queue metadata
2214 - $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
2215 -
2216 - // Determine if queue is complete
2217 - $is_complete = ($pending === 0 && $processing === 0);
2218 -
2219 - // Get last update time
2220 - $last_update = $wpdb->get_var($wpdb->prepare(
2221 - "SELECT MAX(UNIX_TIMESTAMP(COALESCE(completed_at, started_at, created_at)))
2222 - FROM $table_name
2223 - WHERE queue_id = %s",
2224 - $queue_id
2225 - ));
2226 -
2227 - $last_update_text = $last_update ? human_time_diff($last_update, time()) . ' ' . __('ago', 'mxchat') : __('Just now', 'mxchat');
2228 -
2229 - // Format based on type
2230 - if ($type === 'pdf') {
2231 - return array(
2232 - 'total_pages' => $total,
2233 - 'processed_pages' => $completed + $failed,
2234 - 'failed_pages' => $failed,
2235 - 'percentage' => $percentage,
2236 - 'status' => $is_complete ? 'complete' : 'processing',
2237 - 'last_update' => $last_update_text,
2238 - 'failed_pages_list' => $failed_items,
2239 - 'pdf_url' => $source_url,
2240 - 'queue_id' => $queue_id
2241 - );
2242 - } else {
2243 - return array(
2244 - 'total_urls' => $total,
2245 - 'processed_urls' => $completed + $failed,
2246 - 'failed_urls' => $failed,
2247 - 'percentage' => $percentage,
2248 - 'status' => $is_complete ? 'complete' : 'processing',
2249 - 'last_update' => $last_update_text,
2250 - 'failed_urls_list' => $failed_items,
2251 - 'sitemap_url' => $source_url,
2252 - 'queue_id' => $queue_id
2253 - );
2254 - }
2255 -}
2256 -
2257 -/**
2258 - * Public method to get processing status for both sitemap and PDF queues
2259 - * Used by admin pages to display processing status
2260 - *
2261 - * @return array Array with 'sitemap_status', 'pdf_status', and 'is_processing' keys
2262 - */
2263 -public function mxchat_get_processing_statuses() {
2264 - $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
2265 - $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
2266 -
2267 - $sitemap_status = false;
2268 - $pdf_status = false;
2269 -
2270 - if ($sitemap_queue_id) {
2271 - $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
2272 - }
2273 -
2274 - if ($pdf_queue_id) {
2275 - $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
2276 - }
2277 -
2278 - $is_processing =
2279 - ($sitemap_status && $sitemap_status['status'] === 'processing') ||
2280 - ($pdf_status && $pdf_status['status'] === 'processing');
2281 -
2282 - return array(
2283 - 'sitemap_status' => $sitemap_status,
2284 - 'pdf_status' => $pdf_status,
2285 - 'is_processing' => $is_processing
2286 - );
2287 -}
2288 -
2289 -/**
2290 - * AJAX handler to get recent knowledge entries for real-time table updates
2291 - * UPDATED: Now supports both WordPress DB and Pinecone data sources
2292 - */
2293 -public function ajax_mxchat_get_recent_entries() {
2294 - check_ajax_referer('mxchat_entries_nonce', 'nonce');
2295 -
2296 - if (!current_user_can('manage_options')) {
2297 - wp_send_json_error(array('message' => 'Unauthorized'));
2298 - return;
2299 - }
2300 -
2301 - global $wpdb;
2302 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2303 -
2304 - // Get parameters
2305 - $last_id = isset($_POST['last_id']) ? absint($_POST['last_id']) : 0;
2306 - $limit = isset($_POST['limit']) ? min(absint($_POST['limit']), 50) : 10;
2307 - $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2308 -
2309 - // Check if Pinecone is enabled for this bot
2310 - $pinecone_manager = $this->mxchat_get_pinecone_manager();
2311 - $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
2312 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2313 - $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
2314 -
2315 - if ($use_pinecone && $has_pinecone_api) {
2316 - // PINECONE DATA SOURCE - Get grouped entry count (not raw vector count)
2317 - // Use mxchat_fetch_pinecone_records which returns total_unique_entries
2318 - $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, '', 1, 10, $bot_id, '');
2319 - $total_count = $records['total'] ?? 0;
2320 -
2321 - // For Pinecone, we don't return individual entries during polling
2322 - // (entries are already displayed on page load via mxchat_fetch_pinecone_records)
2323 - // We just return the updated count
2324 - wp_send_json_success(array(
2325 - 'entries' => array(),
2326 - 'total_count' => absint($total_count),
2327 - 'max_id' => $last_id,
2328 - 'data_source' => 'pinecone'
2329 - ));
2330 - return;
2331 - }
2332 -
2333 - // WORDPRESS DB DATA SOURCE
2334 - // Build query to get entries newer than last_id
2335 - $where_clauses = array('1=1');
2336 - $where_values = array();
2337 -
2338 - if ($last_id > 0) {
2339 - $where_clauses[] = 'id > %d';
2340 - $where_values[] = $last_id;
2341 - }
2342 -
2343 - // Note: WordPress DB table doesn't have bot_id column
2344 - // Multi-bot filtering is handled via Pinecone namespaces
2345 -
2346 - $where_sql = implode(' AND ', $where_clauses);
2347 -
2348 - // Get recent entries
2349 - $query = "SELECT id, article_content, source_url, timestamp
2350 - FROM $table_name
2351 - WHERE $where_sql
2352 - ORDER BY id DESC
2353 - LIMIT %d";
2354 -
2355 - $where_values[] = $limit;
2356 -
2357 - $entries = $wpdb->get_results($wpdb->prepare($query, $where_values));
2358 -
2359 - // Get total count of GROUPED entries (by source_url) - matches pagination display
2360 - // Count unique source_urls (excluding mxchat:// internal refs) + count of ungrouped rows
2361 - $total_count = $wpdb->get_var(
2362 - "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2363 - (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
2364 - );
2365 -
2366 - // Format entries for response
2367 - $formatted_entries = array();
2368 - $preview_length = 150;
2369 - foreach ($entries as $entry) {
2370 - // Parse chunk metadata using the proper chunker method (same as initial page load)
2371 - if (class_exists('MxChat_Chunker')) {
2372 - $chunk_meta = MxChat_Chunker::parse_stored_chunk($entry->article_content);
2373 - $display_content = $chunk_meta['text'];
2374 - $chunk_metadata = $chunk_meta['metadata'];
2375 - } else {
2376 - $display_content = $entry->article_content;
2377 - $chunk_metadata = array();
2378 - }
2379 -
2380 - $content_preview = mb_strlen($display_content) > $preview_length
2381 - ? mb_substr($display_content, 0, $preview_length) . '...'
2382 - : $display_content;
2383 -
2384 - $formatted_entries[] = array(
2385 - 'id' => $entry->id,
2386 - 'preview' => esc_html($content_preview),
2387 - 'full_content' => wp_kses_post(wpautop($display_content)),
2388 - 'content_length' => mb_strlen($display_content),
2389 - 'preview_length' => $preview_length,
2390 - 'source_url' => $entry->source_url,
2391 - 'has_link' => !empty($entry->source_url) && strpos($entry->source_url, 'mxchat://') !== 0,
2392 - 'chunk_metadata' => $chunk_metadata,
2393 - 'bot_id' => $entry->bot_id ?? 'default',
2394 - 'edit_nonce' => wp_create_nonce('mxchat_edit_entry_nonce'),
2395 - 'delete_nonce' => wp_create_nonce('mxchat_delete_prompt_nonce')
2396 - );
2397 - }
2398 -
2399 - wp_send_json_success(array(
2400 - 'entries' => $formatted_entries,
2401 - 'total_count' => absint($total_count),
2402 - 'max_id' => !empty($entries) ? $entries[0]->id : $last_id,
2403 - 'data_source' => 'wordpress'
2404 - ));
2405 -}
2406 -
2407 -/**
2408 - * Get Pinecone total count from stats API
2409 - * Helper function for ajax_mxchat_get_recent_entries
2410 - */
2411 -private function mxchat_get_pinecone_count_from_stats($pinecone_options) {
2412 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2413 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
2414 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
2415 -
2416 - if (empty($api_key) || empty($host)) {
2417 - return 0;
2418 - }
2419 -
2420 - try {
2421 - $stats_url = "https://{$host}/describe_index_stats";
2422 -
2423 - $response = wp_remote_post($stats_url, array(
2424 - 'headers' => array(
2425 - 'Api-Key' => $api_key,
2426 - 'Content-Type' => 'application/json'
2427 - ),
2428 - 'body' => '{}',
2429 - 'timeout' => 10
2430 - ));
2431 -
2432 - if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
2433 - $body = wp_remote_retrieve_body($response);
2434 - $stats_data = json_decode($body, true);
2435 -
2436 - // If namespace is specified, get count from that specific namespace
2437 - if (!empty($namespace) && isset($stats_data['namespaces'][$namespace]['vectorCount'])) {
2438 - return intval($stats_data['namespaces'][$namespace]['vectorCount']);
2439 - }
2440 -
2441 - // If no namespace specified or namespace not found in response, use total
2442 - return intval($stats_data['totalVectorCount'] ?? 0);
2443 - }
2444 -
2445 - return 0;
2446 -
2447 - } catch (Exception $e) {
2448 - return 0;
2449 - }
2450 -}
2451 -
2452 -/**
2453 - * AJAX handler to refresh Pinecone entries table via AJAX
2454 - * Returns the table HTML for updating the UI without a full page reload
2455 - */
2456 -public function ajax_mxchat_refresh_pinecone_entries() {
2457 - check_ajax_referer('mxchat_entries_nonce', 'nonce');
2458 -
2459 - if (!current_user_can('manage_options')) {
2460 - wp_send_json_error(array('message' => 'Unauthorized'));
2461 - return;
2462 - }
2463 -
2464 - $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2465 - $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2466 - $per_page = 25;
2467 - $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2468 - $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2469 -
2470 - // Get Pinecone manager and options
2471 - $pinecone_manager = $this->mxchat_get_pinecone_manager();
2472 - if (!$pinecone_manager) {
2473 - wp_send_json_error(array('message' => 'Pinecone manager not available'));
2474 - return;
2475 - }
2476 -
2477 - $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
2478 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2479 - $pinecone_api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2480 -
2481 - if (!$use_pinecone || empty($pinecone_api_key)) {
2482 - wp_send_json_error(array('message' => 'Pinecone not configured'));
2483 - return;
2484 - }
2485 -
2486 - // Fetch records from Pinecone
2487 - $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, $search_query, $page, $per_page, $bot_id, $content_type_filter);
2488 - $prompts = $records['data'] ?? array();
2489 - $total_records = $records['total'] ?? 0;
2490 -
2491 - // Preprocess Pinecone records — set chunk_metadata and display_content
2492 - // (matches admin-knowledge-page.php preprocessing)
2493 - foreach ($prompts as $prompt) {
2494 - if (isset($prompt->chunk_index) && $prompt->chunk_index !== null) {
2495 - $prompt->chunk_metadata = array(
2496 - 'chunk_index' => intval($prompt->chunk_index),
2497 - 'total_chunks' => isset($prompt->total_chunks) ? intval($prompt->total_chunks) : null,
2498 - 'is_chunked' => isset($prompt->is_chunked) ? (bool) $prompt->is_chunked : true,
2499 - 'source_url' => $prompt->source_url ?? ''
2500 - );
2501 - $prompt->display_content = $prompt->article_content;
2502 - } else {
2503 - $prompt->chunk_metadata = array();
2504 - $prompt->display_content = $prompt->article_content ?? '';
2505 - }
2506 - }
2507 -
2508 - // Group prompts by source_url
2509 - $grouped_prompts = array();
2510 - foreach ($prompts as $prompt) {
2511 - $source_url = '';
2512 - if (!empty($prompt->chunk_metadata['source_url'])) {
2513 - $source_url = $prompt->chunk_metadata['source_url'];
2514 - } elseif (!empty($prompt->source_url)) {
2515 - $source_url = $prompt->source_url;
2516 - }
2517 -
2518 - if (!empty($source_url)) {
2519 - if (!isset($grouped_prompts[$source_url])) {
2520 - $grouped_prompts[$source_url] = array();
2521 - }
2522 - $grouped_prompts[$source_url][] = $prompt;
2523 - } else {
2524 - $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2525 - }
2526 - }
2527 -
2528 - // Sort each group by chunk_index
2529 - foreach ($grouped_prompts as $source_url => &$group) {
2530 - usort($group, function($a, $b) {
2531 - $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
2532 - $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
2533 - return $index_a - $index_b;
2534 - });
2535 - }
2536 - unset($group);
2537 -
2538 - // Build HTML for the table rows - must match admin-knowledge-page.php structure exactly
2539 - ob_start();
2540 - $display_index = 0;
2541 - $current_page = $page;
2542 - $data_source = 'pinecone';
2543 - $current_bot_id = $bot_id;
2544 - $preview_length = 150;
2545 -
2546 - if (empty($grouped_prompts)) {
2547 - echo '<tr><td colspan="4" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
2548 - esc_html_e('No knowledge entries found in Pinecone.', 'mxchat');
2549 - echo '</td></tr>';
2550 - } else {
2551 - foreach ($grouped_prompts as $source_url => $group) {
2552 - $chunk_count = count($group);
2553 - $first_prompt = $group[0];
2554 - $display_index++;
2555 -
2556 - if ($chunk_count > 1) {
2557 - // Multiple chunks - show grouped row with expand button
2558 - $group_id = 'group-' . md5($source_url);
2559 - ?>
2560 - <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
2561 - class="mxchat-chunk-group-header"
2562 - data-source="<?php echo esc_attr($data_source); ?>"
2563 - data-group-id="<?php echo esc_attr($group_id); ?>"
2564 - style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2565 - <td style="padding: 12px 16px; text-align: center;">
2566 - <input type="checkbox"
2567 - class="mxchat-entry-checkbox"
2568 - data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2569 - data-source="<?php echo esc_attr($data_source); ?>"
2570 - data-source-url="<?php echo esc_attr($source_url); ?>"
2571 - data-is-group="true"
2572 - data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
2573 - </td>
2574 - <td style="padding: 12px 16px; font-size: 13px;">
2575 - <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2576 - </td>
2577 - <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2578 - <div class="mxchat-chunk-group-info">
2579 - <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
2580 - <span class="dashicons dashicons-arrow-right-alt2"></span>
2581 - </button>
2582 - <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
2583 - <span class="mxchat-chunk-preview">
2584 - <?php
2585 - $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : (isset($first_prompt->article_content) ? $first_prompt->article_content : '');
2586 - $content_preview = mb_substr($parent_content, 0, 100);
2587 - echo esc_html($content_preview . '...');
2588 - ?>
2589 - </span>
2590 - </div>
2591 - </td>
2592 - <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2593 - <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
2594 - <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2595 - <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2596 - <?php esc_html_e('View Source', 'mxchat'); ?>
2597 - </a>
2598 - <?php else : ?>
2599 - <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
2600 - <?php endif; ?>
2601 - </td>
2602 - <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
2603 - <?php if ($data_source !== 'pinecone') : ?>
2604 - <button type="button"
2605 - class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
2606 - data-source-url="<?php echo esc_attr($source_url); ?>"
2607 - data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
2608 - data-data-source="<?php echo esc_attr($data_source); ?>"
2609 - data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2610 - data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
2611 - title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
2612 - <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
2613 - </button>
2614 - <?php endif; ?>
2615 - <button type="button"
2616 - class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
2617 - data-source-url="<?php echo esc_attr($source_url); ?>"
2618 - data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
2619 - data-data-source="<?php echo esc_attr($data_source); ?>"
2620 - data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
2621 - data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
2622 - style="color: var(--mxch-error);"
2623 - title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
2624 - <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2625 - </button>
2626 - </td>
2627 - </tr>
2628 - <?php
2629 - // Render hidden chunk rows
2630 - foreach ($group as $chunk_index => $chunk) {
2631 - $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
2632 - $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
2633 - $content = isset($chunk->display_content) ? $chunk->display_content : (isset($chunk->article_content) ? $chunk->article_content : '');
2634 - $content_preview = mb_strlen($content) > $preview_length
2635 - ? mb_substr($content, 0, $preview_length) . '...'
2636 - : $content;
2637 - ?>
2638 - <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
2639 - class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
2640 - data-source="<?php echo esc_attr($data_source); ?>"
2641 - style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
2642 - <td style="padding: 12px 16px; text-align: center;">
2643 - <!-- Checkbox column placeholder for chunks (managed by group) -->
2644 - </td>
2645 - <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
2646 - <!-- Hidden ID column for chunks -->
2647 - </td>
2648 - <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2649 - <div class="mxchat-accordion-wrapper">
2650 - <div class="mxchat-content-preview">
2651 - <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
2652 - <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
2653 - </span>
2654 - <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2655 - <?php if (mb_strlen($content) > $preview_length) : ?>
2656 - <button class="mxchat-expand-toggle" type="button">
2657 - <span class="dashicons dashicons-arrow-down-alt2"></span>
2658 - </button>
2659 - <?php endif; ?>
2660 - </div>
2661 - <div class="mxchat-content-full" style="display: none;">
2662 - <div class="content-view">
2663 - <?php
2664 - if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2665 - echo '<div dir="rtl" lang="he" class="rtl-content">';
2666 - echo wp_kses_post(wpautop($content));
2667 - echo '</div>';
2668 - } else {
2669 - echo wp_kses_post(wpautop($content));
2670 - }
2671 - ?>
2672 - </div>
2673 - </div>
2674 - </div>
2675 - </td>
2676 - <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2677 - <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
2678 - </td>
2679 - <td class="mxchat-actions-cell" style="padding: 12px 16px;">
2680 - <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
2681 - </td>
2682 - </tr>
2683 - <?php
2684 - }
2685 - } else {
2686 - // Single entry - display normally with accordion
2687 - $prompt = $first_prompt;
2688 - $content = isset($prompt->display_content) ? $prompt->display_content : (isset($prompt->article_content) ? $prompt->article_content : '');
2689 - $content_preview = mb_strlen($content) > $preview_length
2690 - ? mb_substr($content, 0, $preview_length) . '...'
2691 - : $content;
2692 - ?>
2693 - <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
2694 - data-source="<?php echo esc_attr($data_source); ?>"
2695 - style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
2696 - <td style="padding: 12px 16px; text-align: center;">
2697 - <input type="checkbox"
2698 - class="mxchat-entry-checkbox"
2699 - data-entry-id="<?php echo esc_attr($prompt->id); ?>"
2700 - data-source="<?php echo esc_attr($data_source); ?>"
2701 - data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
2702 - data-is-group="false"
2703 - data-chunk-count="1">
2704 - </td>
2705 - <td style="padding: 12px 16px; font-size: 13px;">
2706 - <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
2707 - </td>
2708 - <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
2709 - <div class="mxchat-accordion-wrapper">
2710 - <div class="mxchat-content-preview">
2711 - <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
2712 - <?php if (mb_strlen($content) > $preview_length) : ?>
2713 - <button class="mxchat-expand-toggle" type="button">
2714 - <span class="dashicons dashicons-arrow-down-alt2"></span>
2715 - </button>
2716 - <?php endif; ?>
2717 - </div>
2718 - <div class="mxchat-content-full" style="display: none;">
2719 - <div class="content-view">
2720 - <?php
2721 - if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
2722 - echo '<div dir="rtl" lang="he" class="rtl-content">';
2723 - echo wp_kses_post(wpautop($content));
2724 - echo '</div>';
2725 - } else {
2726 - echo wp_kses_post(wpautop($content));
2727 - }
2728 - ?>
2729 - </div>
2730 - </div>
2731 - </div>
2732 - </td>
2733 - <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
2734 - <?php
2735 - $actual_source = $source_url;
2736 - if (strpos($source_url, '_ungrouped_') === 0) {
2737 - $actual_source = $prompt->source_url ?? '';
2738 - }
2739 - if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
2740 - <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
2741 - <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
2742 - <?php esc_html_e('View', 'mxchat'); ?>
2743 - </a>
2744 - <?php else : ?>
2745 - <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
2746 - <?php endif; ?>
2747 - </td>
2748 - <td style="padding: 12px 16px;">
2749 - <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-ajax" data-vector-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_pinecone_prompt_nonce'); ?>" style="color: var(--mxch-error);">
2750 - <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
2751 - </button>
2752 - </td>
2753 - </tr>
2754 - <?php
2755 - }
2756 - }
2757 - }
2758 - $html = ob_get_clean();
2759 -
2760 - // Generate pagination HTML for Pinecone
2761 - $total_pages = ceil($total_records / $per_page);
2762 - $pagination_html = '';
2763 - if ($total_pages > 1) {
2764 - $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
2765 -
2766 - // Previous button
2767 - if ($page > 1) {
2768 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
2769 - }
2770 -
2771 - // Page numbers
2772 - $start_page = max(1, $page - 2);
2773 - $end_page = min($total_pages, $page + 2);
2774 -
2775 - if ($start_page > 1) {
2776 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
2777 - if ($start_page > 2) {
2778 - $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2779 - }
2780 - }
2781 -
2782 - for ($i = $start_page; $i <= $end_page; $i++) {
2783 - if ($i == $page) {
2784 - $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
2785 - } else {
2786 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
2787 - }
2788 - }
2789 -
2790 - if ($end_page < $total_pages) {
2791 - if ($end_page < $total_pages - 1) {
2792 - $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
2793 - }
2794 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
2795 - }
2796 -
2797 - // Next button
2798 - if ($page < $total_pages) {
2799 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
2800 - }
2801 -
2802 - $pagination_html .= '</div>';
2803 - }
2804 -
2805 - wp_send_json_success(array(
2806 - 'html' => $html,
2807 - 'pagination_html' => $pagination_html,
2808 - 'total_count' => $total_records,
2809 - 'total_pages' => $total_pages,
2810 - 'page' => $page,
2811 - 'per_page' => $per_page,
2812 - 'data_source' => 'pinecone'
2813 - ));
2814 -}
2815 -
2816 -/**
2817 - * AJAX handler for pagination - handles both WordPress DB and Pinecone data sources
2818 - * Returns paginated entries without requiring a full page reload
2819 - */
2820 -public function ajax_mxchat_paginate_entries() {
2821 - check_ajax_referer('mxchat_entries_nonce', 'nonce');
2822 -
2823 - if (!current_user_can('manage_options')) {
2824 - wp_send_json_error(array('message' => 'Unauthorized'));
2825 - return;
2826 - }
2827 -
2828 - $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
2829 - $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
2830 - $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
2831 - $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
2832 - $per_page = 25;
2833 -
2834 - // Check if Pinecone is enabled for this bot
2835 - $pinecone_manager = $this->mxchat_get_pinecone_manager();
2836 - $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
2837 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
2838 - $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);
2839 -
2840 - if ($use_pinecone && $has_pinecone_api) {
2841 - // Delegate to Pinecone pagination handler (pass search params)
2842 - $_POST['page'] = $page;
2843 - $_POST['search'] = $search_query;
2844 - $_POST['content_type'] = $content_type_filter;
2845 - $this->ajax_mxchat_refresh_pinecone_entries();
2846 - return;
2847 - }
2848 -
2849 - // WordPress DB pagination - MUST match initial page load logic exactly
2850 - global $wpdb;
2851 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2852 - $offset = ($page - 1) * $per_page;
2853 -
2854 - // Build WHERE clause for search and content type filtering
2855 - $where_clauses = array();
2856 - $where_values = array();
2857 -
2858 - if ($search_query) {
2859 - $where_clauses[] = "article_content LIKE %s";
2860 - $where_values[] = '%' . $wpdb->esc_like($search_query) . '%';
2861 - }
2862 -
2863 - if ($content_type_filter) {
2864 - switch ($content_type_filter) {
2865 - case 'manual':
2866 - $where_clauses[] = "(source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')";
2867 - break;
2868 - case 'pdf':
2869 - $where_clauses[] = "source_url LIKE '%.pdf'";
2870 - break;
2871 - case 'url':
2872 - $where_clauses[] = "source_url != '' AND source_url IS NOT NULL AND source_url NOT LIKE 'mxchat://%' AND source_url NOT LIKE '%.pdf'";
2873 - break;
2874 - }
2875 - }
2876 -
2877 - $where_sql = !empty($where_clauses) ? 'WHERE ' . implode(' AND ', $where_clauses) : '';
2878 -
2879 - // Count grouped entries with filters applied
2880 - if (!empty($where_values)) {
2881 - $count_args = array_merge($where_values, $where_values);
2882 - $count_query = $wpdb->prepare(
2883 - "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2884 - (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))",
2885 - ...$count_args
2886 - );
2887 - $total_records = $wpdb->get_var($count_query);
2888 - } else if (!empty($where_sql)) {
2889 - // Content type filter only (no search), no prepared values needed
2890 - $total_records = $wpdb->get_var(
2891 - "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2892 - (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))"
2893 - );
2894 - } else {
2895 - // No filters
2896 - $total_records = $wpdb->get_var(
2897 - "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
2898 - (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
2899 - );
2900 - }
2901 - $total_pages = ceil($total_records / $per_page);
2902 -
2903 - // Step 1: Get unique source_urls for this page (ordered by latest timestamp) with filters
2904 - if (!empty($where_values)) {
2905 - $query_args = array_merge($where_values, array($per_page, $offset));
2906 - $urls_query = $wpdb->prepare(
2907 - "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2908 - {$where_sql}
2909 - GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2910 - ...$query_args
2911 - );
2912 - } else if (!empty($where_sql)) {
2913 - $urls_query = $wpdb->prepare(
2914 - "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2915 - {$where_sql}
2916 - GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2917 - $per_page, $offset
2918 - );
2919 - } else {
2920 - $urls_query = $wpdb->prepare(
2921 - "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
2922 - GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
2923 - $per_page, $offset
2924 - );
2925 - }
2926 - $page_urls = $wpdb->get_results($urls_query);
2927 -
2928 - // Step 2: Build list of source_urls to fetch
2929 - $url_list = array();
2930 - $url_order_map = array();
2931 - $order_index = 0;
2932 - foreach ($page_urls as $url_row) {
2933 - $url = $url_row->source_url;
2934 - $url_list[] = $url;
2935 - $url_order_map[$url] = $order_index++;
2936 - }
2937 -
2938 - // Step 3: Fetch all rows for these source_urls (with search filter if applicable)
2939 - $prompts = array();
2940 - if (!empty($url_list)) {
2941 - $placeholders = implode(',', array_fill(0, count($url_list), '%s'));
2942 - if ($search_query) {
2943 - // Include search filter in the final fetch
2944 - $prompts_query = $wpdb->prepare(
2945 - "SELECT id, article_content, source_url, timestamp, role_restriction
2946 - FROM {$table_name}
2947 - WHERE source_url IN ($placeholders) AND article_content LIKE %s
2948 - ORDER BY timestamp DESC",
2949 - ...array_merge($url_list, ['%' . $wpdb->esc_like($search_query) . '%'])
2950 - );
2951 - } else {
2952 - $prompts_query = $wpdb->prepare(
2953 - "SELECT id, article_content, source_url, timestamp, role_restriction
2954 - FROM {$table_name}
2955 - WHERE source_url IN ($placeholders)
2956 - ORDER BY timestamp DESC",
2957 - $url_list
2958 - );
2959 - }
2960 - $prompts = $wpdb->get_results($prompts_query);
2961 - }
2962 -
2963 - // Group prompts by source_url for chunk display
2964 - $grouped_prompts = array();
2965 - foreach ($prompts as $prompt) {
2966 - $source_url = $prompt->source_url ?? '';
2967 -
2968 - // Parse chunk metadata using the proper chunker method (same as initial page load)
2969 - if (class_exists('MxChat_Chunker')) {
2970 - $chunk_meta = MxChat_Chunker::parse_stored_chunk($prompt->article_content);
2971 - $prompt->chunk_metadata = $chunk_meta['metadata'];
2972 - $prompt->display_content = $chunk_meta['text'];
2973 - } else {
2974 - $prompt->chunk_metadata = array();
2975 - $prompt->display_content = $prompt->article_content;
2976 - }
2977 -
2978 - if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0) {
2979 - if (!isset($grouped_prompts[$source_url])) {
2980 - $grouped_prompts[$source_url] = array();
2981 - }
2982 - $grouped_prompts[$source_url][] = $prompt;
2983 - } else {
2984 - // Ungrouped entries
2985 - $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
2986 - }
2987 - }
2988 -
2989 - // Sort groups by the original URL order (newest first)
2990 - uksort($grouped_prompts, function($a, $b) use ($url_order_map) {
2991 - $order_a = isset($url_order_map[$a]) ? $url_order_map[$a] : PHP_INT_MAX;
2992 - $order_b = isset($url_order_map[$b]) ? $url_order_map[$b] : PHP_INT_MAX;
2993 - return $order_a - $order_b;
2994 - });
2995 -
2996 - // Sort each group internally by chunk_index
2997 - foreach ($grouped_prompts as $source_url => &$group) {
2998 - usort($group, function($a, $b) {
2999 - $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
3000 - $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
3001 - return $index_a - $index_b;
3002 - });
3003 - }
3004 - unset($group);
3005 -
3006 - // Build HTML for the table rows
3007 - ob_start();
3008 - $display_index = 0;
3009 - $current_page = $page;
3010 - $data_source = 'wordpress';
3011 - $current_bot_id = $bot_id;
3012 - $preview_length = 150;
3013 -
3014 - if (empty($grouped_prompts)) {
3015 - echo '<tr><td colspan="5" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
3016 - esc_html_e('No knowledge entries found. Use the Import Options to add content.', 'mxchat');
3017 - echo '</td></tr>';
3018 - } else {
3019 - foreach ($grouped_prompts as $source_url => $group) {
3020 - $chunk_count = count($group);
3021 - $first_prompt = $group[0];
3022 - $display_index++;
3023 -
3024 - if ($chunk_count > 1) {
3025 - // Multiple chunks - show grouped row with expand button
3026 - $group_id = 'group-' . md5($source_url);
3027 - ?>
3028 - <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
3029 - class="mxchat-chunk-group-header"
3030 - data-source="<?php echo esc_attr($data_source); ?>"
3031 - data-group-id="<?php echo esc_attr($group_id); ?>"
3032 - style="border-bottom: 1px solid var(--mxch-card-border);">
3033 - <td style="padding: 12px 16px; text-align: center;">
3034 - <input type="checkbox"
3035 - class="mxchat-entry-checkbox"
3036 - data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
3037 - data-source="<?php echo esc_attr($data_source); ?>"
3038 - data-source-url="<?php echo esc_attr($source_url); ?>"
3039 - data-is-group="true"
3040 - data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
3041 - </td>
3042 - <td style="padding: 12px 16px; font-size: 13px;">
3043 - <?php echo esc_html($first_prompt->id); ?>
3044 - </td>
3045 - <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3046 - <div class="mxchat-chunk-group-info">
3047 - <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
3048 - <span class="dashicons dashicons-arrow-right-alt2"></span>
3049 - </button>
3050 - <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
3051 - <span class="mxchat-chunk-preview">
3052 - <?php
3053 - $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : $first_prompt->article_content;
3054 - $content_preview = mb_substr($parent_content, 0, 100);
3055 - echo esc_html($content_preview . '...');
3056 - ?>
3057 - </span>
3058 - </div>
3059 - </td>
3060 - <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3061 - <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
3062 - <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
3063 - <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
3064 - <?php esc_html_e('View Source', 'mxchat'); ?>
3065 - </a>
3066 - <?php else : ?>
3067 - <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
3068 - <?php endif; ?>
3069 - </td>
3070 - <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
3071 - <?php if ($data_source !== 'pinecone') : ?>
3072 - <button type="button"
3073 - class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
3074 - data-source-url="<?php echo esc_attr($source_url); ?>"
3075 - data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
3076 - data-data-source="<?php echo esc_attr($data_source); ?>"
3077 - data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3078 - data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
3079 - title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
3080 - <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
3081 - </button>
3082 - <?php endif; ?>
3083 - <button type="button"
3084 - class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
3085 - data-source-url="<?php echo esc_attr($source_url); ?>"
3086 - data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
3087 - data-data-source="<?php echo esc_attr($data_source); ?>"
3088 - data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3089 - data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
3090 - style="color: var(--mxch-error);"
3091 - title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
3092 - <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
3093 - </button>
3094 - </td>
3095 - </tr>
3096 - <?php
3097 - // Render hidden chunk rows
3098 - foreach ($group as $chunk_index => $chunk) {
3099 - $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
3100 - $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
3101 - $content = isset($chunk->display_content) ? $chunk->display_content : $chunk->article_content;
3102 - $content_preview = mb_strlen($content) > $preview_length
3103 - ? mb_substr($content, 0, $preview_length) . '...'
3104 - : $content;
3105 - ?>
3106 - <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
3107 - class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
3108 - data-source="<?php echo esc_attr($data_source); ?>"
3109 - style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
3110 - <td style="padding: 12px 16px; text-align: center;">
3111 - <!-- Checkbox column placeholder for chunks (managed by group) -->
3112 - </td>
3113 - <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
3114 - <!-- Hidden ID column for chunks -->
3115 - </td>
3116 - <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3117 - <div class="mxchat-accordion-wrapper">
3118 - <div class="mxchat-content-preview">
3119 - <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
3120 - <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
3121 - </span>
3122 - <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
3123 - <?php if (mb_strlen($content) > $preview_length) : ?>
3124 - <button class="mxchat-expand-toggle" type="button">
3125 - <span class="dashicons dashicons-arrow-down-alt2"></span>
3126 - </button>
3127 - <?php endif; ?>
3128 - </div>
3129 - <div class="mxchat-content-full" style="display: none;">
3130 - <div class="content-view">
3131 - <?php
3132 - if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
3133 - echo '<div dir="rtl" lang="he" class="rtl-content">';
3134 - echo wp_kses_post(wpautop($content));
3135 - echo '</div>';
3136 - } else {
3137 - echo wp_kses_post(wpautop($content));
3138 - }
3139 - ?>
3140 - </div>
3141 - </div>
3142 - </div>
3143 - </td>
3144 - <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3145 - <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
3146 - </td>
3147 - <td class="mxchat-actions-cell" style="padding: 12px 16px;">
3148 - <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
3149 - </td>
3150 - </tr>
3151 - <?php
3152 - }
3153 - } else {
3154 - // Single entry - display normally with accordion
3155 - $prompt = $first_prompt;
3156 - $content = isset($prompt->display_content) ? $prompt->display_content : $prompt->article_content;
3157 - $content_preview = mb_strlen($content) > $preview_length
3158 - ? mb_substr($content, 0, $preview_length) . '...'
3159 - : $content;
3160 - ?>
3161 - <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
3162 - data-source="<?php echo esc_attr($data_source); ?>"
3163 - style="border-bottom: 1px solid var(--mxch-card-border);">
3164 - <td style="padding: 12px 16px; text-align: center;">
3165 - <input type="checkbox"
3166 - class="mxchat-entry-checkbox"
3167 - data-entry-id="<?php echo esc_attr($prompt->id); ?>"
3168 - data-source="<?php echo esc_attr($data_source); ?>"
3169 - data-source-url="<?php echo esc_attr($source_url); ?>"
3170 - data-is-group="false">
3171 - </td>
3172 - <td style="padding: 12px 16px; font-size: 13px;">
3173 - <?php echo esc_html($prompt->id); ?>
3174 - </td>
3175 - <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
3176 - <div class="mxchat-accordion-wrapper">
3177 - <div class="mxchat-content-preview">
3178 - <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
3179 - <?php if (mb_strlen($content) > $preview_length) : ?>
3180 - <button class="mxchat-expand-toggle" type="button">
3181 - <span class="dashicons dashicons-arrow-down-alt2"></span>
3182 - </button>
3183 - <?php endif; ?>
3184 - </div>
3185 - <div class="mxchat-content-full" style="display: none;">
3186 - <div class="content-view">
3187 - <?php
3188 - if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
3189 - echo '<div dir="rtl" lang="he" class="rtl-content">';
3190 - echo wp_kses_post(wpautop($content));
3191 - echo '</div>';
3192 - } else {
3193 - echo wp_kses_post(wpautop($content));
3194 - }
3195 - ?>
3196 - </div>
3197 - </div>
3198 - </div>
3199 - </td>
3200 - <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
3201 - <?php
3202 - $actual_source = $source_url;
3203 - if (strpos($source_url, '_ungrouped_') === 0) {
3204 - $actual_source = $prompt->source_url ?? '';
3205 - }
3206 - if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
3207 - <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
3208 - <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
3209 - <?php esc_html_e('View', 'mxchat'); ?>
3210 - </a>
3211 - <?php else : ?>
3212 - <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
3213 - <?php endif; ?>
3214 - </td>
3215 - <td style="padding: 12px 16px; white-space: nowrap;">
3216 - <button type="button"
3217 - class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
3218 - data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
3219 - data-entry-id="<?php echo esc_attr($prompt->id); ?>"
3220 - data-data-source="<?php echo esc_attr($data_source); ?>"
3221 - data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
3222 - data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
3223 - title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
3224 - <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
3225 - </button>
3226 - <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-wordpress" data-entry-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_wordpress_prompt_nonce'); ?>" style="color: var(--mxch-error);">
3227 - <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
3228 - </button>
3229 - </td>
3230 - </tr>
3231 - <?php
3232 - }
3233 - }
3234 - }
3235 - $html = ob_get_clean();
3236 -
3237 - // Generate pagination HTML (include search/filter data for subsequent pages)
3238 - $pagination_html = '';
3239 - if ($total_pages > 1) {
3240 - $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';
3241 -
3242 - // Previous button
3243 - if ($page > 1) {
3244 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
3245 - }
3246 -
3247 - // Page numbers
3248 - $start_page = max(1, $page - 2);
3249 - $end_page = min($total_pages, $page + 2);
3250 -
3251 - if ($start_page > 1) {
3252 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
3253 - if ($start_page > 2) {
3254 - $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
3255 - }
3256 - }
3257 -
3258 - for ($i = $start_page; $i <= $end_page; $i++) {
3259 - if ($i == $page) {
3260 - $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
3261 - } else {
3262 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
3263 - }
3264 - }
3265 -
3266 - if ($end_page < $total_pages) {
3267 - if ($end_page < $total_pages - 1) {
3268 - $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
3269 - }
3270 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
3271 - }
3272 -
3273 - // Next button
3274 - if ($page < $total_pages) {
3275 - $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
3276 - }
3277 -
3278 - $pagination_html .= '</div>';
3279 - }
3280 -
3281 - wp_send_json_success(array(
3282 - 'html' => $html,
3283 - 'pagination_html' => $pagination_html,
3284 - 'total_count' => $total_records,
3285 - 'total_pages' => $total_pages,
3286 - 'page' => $page,
3287 - 'per_page' => $per_page,
3288 - 'data_source' => 'wordpress'
3289 - ));
3290 -}
3291 -
3292 -/**
3293 - * AJAX handler to detect available sitemaps on the site
3294 - * Optimized for speed - only checks primary sitemap indexes first
3295 - */
3296 -public function ajax_mxchat_detect_sitemaps() {
3297 - check_ajax_referer('mxchat_detect_sitemaps_nonce', 'nonce');
3298 -
3299 - if (!current_user_can('manage_options')) {
3300 - wp_send_json_error(array('message' => 'Unauthorized'));
3301 - return;
3302 - }
3303 -
3304 - $site_url = get_site_url();
3305 - $sitemaps = array();
3306 - $found_index = false;
3307 -
3308 - // Only check the main sitemap index files first (much faster)
3309 - // These are the primary entry points that contain sub-sitemaps
3310 - $primary_indexes = array(
3311 - 'sitemap_index.xml' => 'Yoast SEO / Rank Math', // Yoast & Rank Math
3312 - 'wp-sitemap.xml' => 'WordPress Core', // WordPress Core
3313 - 'sitemap.xml' => 'Standard', // Generic/AIOSEO
3314 - );
3315 -
3316 - foreach ($primary_indexes as $path => $source) {
3317 - $url = trailingslashit($site_url) . $path;
3318 -
3319 - $response = wp_remote_head($url, array(
3320 - 'timeout' => 10,
3321 - 'sslverify' => false,
3322 - 'redirection' => 1,
3323 - 'user-agent' => mxchat_ingest_user_agent(),
3324 - ));
3325 -
3326 - if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3327 - // Found a sitemap index - parse it to get sub-sitemaps
3328 - $sub_sitemaps = $this->parse_sitemap_index($url);
3329 - if (!empty($sub_sitemaps)) {
3330 - $sitemaps[] = array(
3331 - 'url' => $url,
3332 - 'type' => 'index',
3333 - 'source' => $source,
3334 - 'sub_sitemaps' => $sub_sitemaps
3335 - );
3336 - $found_index = true;
3337 - // Found a valid index, no need to check others
3338 - break;
3339 - }
3340 - }
3341 - }
3342 -
3343 - // If no sitemap index found, check for standalone sitemaps
3344 - if (!$found_index) {
3345 - $standalone_sitemaps = array(
3346 - 'post-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
3347 - 'page-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
3348 - );
3349 -
3350 - foreach ($standalone_sitemaps as $path => $info) {
3351 - $url = trailingslashit($site_url) . $path;
3352 -
3353 - $response = wp_remote_head($url, array(
3354 - 'timeout' => 2,
3355 - 'sslverify' => false
3356 - ));
3357 -
3358 - if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
3359 - $sitemaps[] = array(
3360 - 'url' => $url,
3361 - 'type' => $info['type'],
3362 - 'source' => $info['source'],
3363 - 'url_count' => 0 // Skip URL count for speed
3364 - );
3365 - }
3366 - }
3367 - }
3368 -
3369 - wp_send_json_success(array(
3370 - 'sitemaps' => $sitemaps,
3371 - 'site_url' => $site_url
3372 - ));
3373 -}
3374 -
3375 -/**
3376 - * Parse a sitemap index to get sub-sitemaps
3377 - * Optimized: doesn't fetch URL count for each sub-sitemap (too slow)
3378 - */
3379 -private function parse_sitemap_index($url) {
3380 - $sub_sitemaps = array();
3381 -
3382 - $response = wp_remote_get($url, array(
3383 - 'timeout' => 30,
3384 - 'sslverify' => false,
3385 - 'user-agent' => mxchat_ingest_user_agent(),
3386 - 'headers' => array(
3387 - 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
3388 - ),
3389 - ));
3390 -
3391 - if (is_wp_error($response)) {
3392 - return $sub_sitemaps;
3393 - }
3394 -
3395 - $body = wp_remote_retrieve_body($response);
3396 - if (empty($body)) {
3397 - return $sub_sitemaps;
3398 - }
3399 -
3400 - // Suppress XML errors
3401 - libxml_use_internal_errors(true);
3402 - $xml = simplexml_load_string($body);
3403 - libxml_clear_errors();
3404 -
3405 - if ($xml === false) {
3406 - return $sub_sitemaps;
3407 - }
3408 -
3409 - // Check if it's a sitemap index (contains <sitemap> elements)
3410 - if (isset($xml->sitemap)) {
3411 - foreach ($xml->sitemap as $sitemap) {
3412 - $loc = (string) $sitemap->loc;
3413 - if (!empty($loc)) {
3414 - // Try to determine the type from the URL
3415 - $type = 'content';
3416 - if (strpos($loc, 'category') !== false || strpos($loc, 'tag') !== false || strpos($loc, 'taxonomy') !== false) {
3417 - $type = 'taxonomy';
3418 - } elseif (strpos($loc, 'author') !== false || strpos($loc, 'user') !== false) {
3419 - $type = 'author';
3420 - }
3421 -
3422 - // Skip URL count - too slow to fetch for each sitemap
3423 - $sub_sitemaps[] = array(
3424 - 'url' => $loc,
3425 - 'type' => $type,
3426 - 'url_count' => 0, // Don't fetch - takes too long
3427 - 'name' => basename(parse_url($loc, PHP_URL_PATH))
3428 - );
3429 - }
3430 - }
3431 - }
3432 -
3433 - return $sub_sitemaps;
3434 -}
3435 -
3436 -/**
3437 - * Get URL count from a sitemap
3438 - */
3439 -private function get_sitemap_url_count($url) {
3440 - $response = wp_remote_get($url, array(
3441 - 'timeout' => 30,
3442 - 'sslverify' => false,
3443 - 'user-agent' => mxchat_ingest_user_agent(),
3444 - 'headers' => array(
3445 - 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
3446 - ),
3447 - ));
3448 -
3449 - if (is_wp_error($response)) {
3450 - return 0;
3451 - }
3452 -
3453 - $body = wp_remote_retrieve_body($response);
3454 - if (empty($body)) {
3455 - return 0;
3456 - }
3457 -
3458 - // Count <url> or <loc> elements
3459 - $count = preg_match_all('/<url>/i', $body, $matches);
3460 - return $count ?: 0;
3461 -}
3462 -
3463 -/**
3464 - * Get sitemaps declared in robots.txt
3465 - */
3466 -private function get_sitemaps_from_robots($site_url) {
3467 - $sitemaps = array();
3468 - $robots_url = trailingslashit($site_url) . 'robots.txt';
3469 -
3470 - $response = wp_remote_get($robots_url, array(
3471 - 'timeout' => 15,
3472 - 'sslverify' => false,
3473 - 'user-agent' => mxchat_ingest_user_agent(),
3474 - ));
3475 -
3476 - if (is_wp_error($response)) {
3477 - return $sitemaps;
3478 - }
3479 -
3480 - $body = wp_remote_retrieve_body($response);
3481 - if (empty($body)) {
3482 - return $sitemaps;
3483 - }
3484 -
3485 - // Find Sitemap: declarations
3486 - if (preg_match_all('/^Sitemap:\s*(.+)$/mi', $body, $matches)) {
3487 - foreach ($matches[1] as $sitemap_url) {
3488 - $sitemap_url = trim($sitemap_url);
3489 - if (filter_var($sitemap_url, FILTER_VALIDATE_URL)) {
3490 - $sitemaps[] = $sitemap_url;
3491 - }
3492 - }
3493 - }
3494 -
3495 - return $sitemaps;
3496 -}
3497 -
3498 1777 public function mxchat_stop_processing() {
3499 1778 // Verify permissions
3500 1779 if (!current_user_can('manage_options')) {
3501 1780 wp_die(esc_html__('Unauthorized access', 'mxchat'));
@@ -3503,52 +1782,28 @@
3503 1782
3504 1783 // Verify nonce
3505 1784 check_admin_referer('mxchat_stop_processing_action', 'mxchat_stop_processing_nonce');
3506 1785
3507 - global $wpdb;
3508 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
3509 -
3510 - // Get active queue IDs
3511 - $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
3512 - $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
3513 -
3514 - // Delete all pending items from active queues
3515 - if ($sitemap_queue_id) {
3516 - $wpdb->delete(
3517 - $table_name,
3518 - array(
3519 - 'queue_id' => $sitemap_queue_id,
3520 - 'status' => 'pending'
3521 - ),
3522 - array('%s', '%s')
3523 - );
3524 -
3525 - delete_transient('mxchat_active_queue_sitemap');
1786 + // Get the last sitemap URL and clear its transient
1787 + $sitemap_url = get_transient('mxchat_last_sitemap_url');
1788 + if ($sitemap_url) {
1789 + delete_transient('mxchat_sitemap_status_' . md5($sitemap_url));
3526 1790 delete_transient('mxchat_last_sitemap_url');
3527 1791 }
3528 -
3529 - if ($pdf_queue_id) {
3530 - // Get PDF path before deleting
3531 - $pdf_path = $this->mxchat_get_queue_meta($pdf_queue_id, 'pdf_path');
3532 -
3533 - $wpdb->delete(
3534 - $table_name,
3535 - array(
3536 - 'queue_id' => $pdf_queue_id,
3537 - 'status' => 'pending'
3538 - ),
3539 - array('%s', '%s')
3540 - );
3541 -
3542 - // Delete PDF file
3543 - if ($pdf_path && file_exists($pdf_path)) {
3544 - wp_delete_file($pdf_path);
3545 - }
3546 -
3547 - delete_transient('mxchat_active_queue_pdf');
1792 +
1793 + // Get the last PDF URL and clear its transient
1794 + $pdf_url = get_transient('mxchat_last_pdf_url');
1795 + if ($pdf_url) {
1796 + delete_transient('mxchat_pdf_status_' . md5($pdf_url));
3548 1797 delete_transient('mxchat_last_pdf_url');
3549 1798 }
3550 1799
1800 + // Unschedule any pending sitemap events
1801 + $timestamp = wp_next_scheduled('mxchat_process_sitemap_urls');
1802 + if ($timestamp) {
1803 + wp_unschedule_event($timestamp, 'mxchat_process_sitemap_urls');
1804 + }
1805 +
3551 1806 // Redirect back with a success message
3552 1807 set_transient('mxchat_admin_notice_success',
3553 1808 esc_html__('Processing has been stopped successfully.', 'mxchat'),
3554 1809 30
@@ -3555,12 +1810,8 @@
3555 1810 );
3556 1811 wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
3557 1812 exit;
3558 1813 }
3559 -
3560 -/**
3561 - * Get content list for processing
3562 - */
3563 1814 public function ajax_mxchat_get_content_list() {
3564 1815 // Verify the nonce
3565 1816 check_ajax_referer('mxchat_content_selector_nonce', 'nonce');
3566 1817
@@ -3568,9 +1819,9 @@
3568 1819 wp_send_json_error(__('Unauthorized access', 'mxchat'));
3569 1820 }
3570 1821
3571 1822 $page = isset($_GET['page']) ? absint($_GET['page']) : 1;
3572 - $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 100;
1823 + $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 20;
3573 1824 $search = isset($_GET['search']) ? sanitize_text_field($_GET['search']) : '';
3574 1825 $post_type = isset($_GET['post_type']) ? sanitize_text_field($_GET['post_type']) : 'all';
3575 1826 $post_status = isset($_GET['post_status']) ? sanitize_text_field($_GET['post_status']) : 'publish';
3576 1827 $processed_filter = isset($_GET['processed_filter']) ? sanitize_text_field($_GET['processed_filter']) : 'all';
@@ -3583,47 +1834,20 @@
3583 1834 'orderby' => 'date',
3584 1835 'order' => 'DESC',
3585 1836 );
3586 1837
3587 - // Handle post types - IMPROVED VERSION
1838 + // Handle post types
3588 1839 if ($post_type !== 'all') {
3589 1840 $args['post_type'] = $post_type;
3590 1841 } else {
3591 - // Get all available post types that might contain content
3592 - $all_post_types = array();
1842 + // Default to post and page if we can't get post types
1843 + $args['post_type'] = array('post', 'page');
3593 1844
3594 - // First get all public post types
3595 - $public_types = get_post_types(array('public' => true), 'names');
3596 - $all_post_types = array_merge($all_post_types, $public_types);
3597 -
3598 - // Add common forum/community post types
3599 - $forum_types = array('topic', 'reply', 'forum', 'wpforo_topic', 'wpforo_post');
3600 - foreach ($forum_types as $forum_type) {
3601 - if (post_type_exists($forum_type)) {
3602 - $all_post_types[] = $forum_type;
3603 - }
1845 + // Try to get public post types
1846 + $public_types = $this->mxchat_get_public_post_types();
1847 + if (is_array($public_types) && !empty($public_types)) {
1848 + $args['post_type'] = array_keys($public_types);
3604 1849 }
3605 -
3606 - // Add other commonly used post types
3607 - $common_types = array('product', 'job_listing', 'event', 'portfolio');
3608 - foreach ($common_types as $common_type) {
3609 - if (post_type_exists($common_type)) {
3610 - $all_post_types[] = $common_type;
3611 - }
3612 - }
3613 -
3614 - // Remove duplicates and ensure we have at least some post types
3615 - $all_post_types = array_unique($all_post_types);
3616 -
3617 - if (empty($all_post_types)) {
3618 - // Fallback to basic post types
3619 - $all_post_types = array('post', 'page');
3620 - }
3621 -
3622 - $args['post_type'] = $all_post_types;
3623 -
3624 - // Debug logging to see what post types are being queried
3625 - //error_log('MxChat Debug: Querying post types: ' . implode(', ', $all_post_types));
3626 1850 }
3627 1851
3628 1852 if (!empty($search)) {
3629 1853 $args['s'] = $search;
@@ -3628,58 +1852,37 @@
3628 1852 if (!empty($search)) {
3629 1853 $args['s'] = $search;
3630 1854 }
3631 1855
3632 - // Get processed data from storage
1856 + // ================================
1857 + // FIXED: Check only the ACTIVE storage method
1858 + // ================================
1859 +
3633 1860 $processed_data = array();
3634 1861
1862 + $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
3635 1863 $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
1864 + $pinecone_manager->mxchat_refresh_after_new_content($pinecone_options);
3636 1865 $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
3637 1866
3638 1867 if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3639 - // Get fresh data from Pinecone - no caching
1868 + // ONLY check Pinecone if it's enabled
3640 1869 $processed_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
3641 1870 } else {
3642 - // WordPress DB checking with better URL matching for all post types
1871 + // ONLY check WordPress DB if Pinecone is not enabled
3643 1872 global $wpdb;
3644 1873 $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3645 - $processed_items = $wpdb->get_results("SELECT id, source_url, article_content, timestamp FROM {$table_name}");
3646 -
3647 - // Group items by source_url to count chunks
3648 - $url_chunk_counts = array();
3649 - $url_latest_timestamp = array();
3650 - $url_first_id = array();
3651 -
1874 + $processed_items = $wpdb->get_results("SELECT id, source_url, timestamp FROM {$table_name}");
1875 +
3652 1876 if (!empty($processed_items)) {
3653 1877 foreach ($processed_items as $item) {
3654 - $url = $item->source_url;
3655 - if (empty($url)) continue;
3656 -
3657 - // Count chunks per URL
3658 - if (!isset($url_chunk_counts[$url])) {
3659 - $url_chunk_counts[$url] = 0;
3660 - $url_latest_timestamp[$url] = $item->timestamp;
3661 - $url_first_id[$url] = $item->id;
3662 - }
3663 - $url_chunk_counts[$url]++;
3664 -
3665 - // Track latest timestamp
3666 - if (strtotime($item->timestamp) > strtotime($url_latest_timestamp[$url])) {
3667 - $url_latest_timestamp[$url] = $item->timestamp;
3668 - }
3669 - }
3670 -
3671 - // Now build processed_data with chunk counts
3672 - foreach ($url_chunk_counts as $url => $chunk_count) {
3673 - $post_id = $this->mxchat_url_to_post_id_improved($url);
3674 -
1878 + $post_id = url_to_postid($item->source_url);
3675 1879 if ($post_id) {
3676 1880 $processed_data[$post_id] = array(
3677 - 'db_id' => $url_first_id[$url],
3678 - 'timestamp' => $url_latest_timestamp[$url],
3679 - 'url' => $url,
3680 - 'source' => 'wordpress',
3681 - 'chunk_count' => $chunk_count
1881 + 'db_id' => $item->id,
1882 + 'timestamp' => $item->timestamp,
1883 + 'url' => $item->source_url,
1884 + 'source' => 'wordpress'
3682 1885 );
3683 1886 }
3684 1887 }
3685 1888 }
@@ -3684,8 +1887,10 @@
3684 1887 }
3685 1888 }
3686 1889 }
3687 1890
1891 + // ================================
1892 +
3688 1893 // Get processed IDs as a simple array for in_array checks
3689 1894 $processed_ids = array_keys($processed_data);
3690 1895
3691 1896 // Handle processed/unprocessed filter
@@ -3727,14 +1932,8 @@
3727 1932 $db_record_id = $item_data['db_id'];
3728 1933 }
3729 1934 }
3730 1935
3731 - // Get chunk count for this item
3732 - $chunk_count = 0;
3733 - if ($is_processed && isset($processed_data[$id]['chunk_count'])) {
3734 - $chunk_count = intval($processed_data[$id]['chunk_count']);
3735 - }
3736 -
3737 1936 $content_items[] = array(
3738 1937 'id' => $id,
3739 1938 'title' => get_the_title(),
3740 1939 'permalink' => get_permalink(),
@@ -3745,10 +1944,9 @@
3745 1944 'word_count' => $word_count,
3746 1945 'already_processed' => $is_processed,
3747 1946 'processed_date' => $processed_date,
3748 1947 'db_record_id' => $db_record_id,
3749 - 'data_source' => $data_source,
3750 - 'chunk_count' => $chunk_count
1948 + 'data_source' => $data_source
3751 1949 );
3752 1950 }
3753 1951 wp_reset_postdata();
3754 1952 }
@@ -3764,178 +1962,8 @@
3764 1962 wp_send_json_success($response);
3765 1963 exit;
3766 1964 }
3767 1965
3768 -
3769 -/**
3770 - * This function handles various WooCommerce URL formats and permalink structures
3771 - */
3772 -private function mxchat_url_to_post_id_improved($url) {
3773 - // First try the standard WordPress function
3774 - $post_id = url_to_postid($url);
3775 -
3776 - if ($post_id > 0) {
3777 - return $post_id;
3778 - }
3779 -
3780 - // If that fails, try more aggressive URL matching
3781 - // Remove trailing slashes and query parameters for better matching
3782 - $clean_url = rtrim($url, '/');
3783 - $clean_url = strtok($clean_url, '?'); // Remove query parameters
3784 -
3785 - // Try again with cleaned URL
3786 - $post_id = url_to_postid($clean_url);
3787 - if ($post_id > 0) {
3788 - return $post_id;
3789 - }
3790 -
3791 - // For bbPress forum topics, try extracting slug from URL
3792 - if (strpos($url, '/topic/') !== false || strpos($url, '/forums/') !== false) {
3793 - // Handle bbPress URLs: /forums/topic/topic-name/
3794 - if (preg_match('/\/forums\/topic\/([^\/\?]+)/', $url, $matches)) {
3795 - $topic_slug = $matches[1];
3796 -
3797 - // Look up topic by slug
3798 - $topic = get_page_by_path($topic_slug, OBJECT, 'topic');
3799 - if ($topic) {
3800 - return $topic->ID;
3801 - }
3802 -
3803 - // Alternative method: query by post_name
3804 - global $wpdb;
3805 - $post_id = $wpdb->get_var($wpdb->prepare(
3806 - "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3807 - $topic_slug
3808 - ));
3809 -
3810 - if ($post_id) {
3811 - return intval($post_id);
3812 - }
3813 - }
3814 -
3815 - // Handle simpler topic URLs: /topic/topic-name/
3816 - if (preg_match('/\/topic\/([^\/\?]+)/', $url, $matches)) {
3817 - $topic_slug = $matches[1];
3818 -
3819 - global $wpdb;
3820 - $post_id = $wpdb->get_var($wpdb->prepare(
3821 - "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
3822 - $topic_slug
3823 - ));
3824 -
3825 - if ($post_id) {
3826 - return intval($post_id);
3827 - }
3828 - }
3829 - }
3830 -
3831 - // For WooCommerce products
3832 - if (strpos($url, '/product/') !== false || strpos($url, 'product=') !== false) {
3833 - // Extract product slug from various URL formats
3834 - $product_slug = '';
3835 -
3836 - // Handle pretty permalinks: /product/product-name/
3837 - if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
3838 - $product_slug = $matches[1];
3839 - }
3840 - // Handle query parameters: ?product=product-name
3841 - elseif (preg_match('/[\?&]product=([^&]+)/', $url, $matches)) {
3842 - $product_slug = $matches[1];
3843 - }
3844 -
3845 - if (!empty($product_slug)) {
3846 - // Look up product by slug
3847 - $product = get_page_by_path($product_slug, OBJECT, 'product');
3848 - if ($product) {
3849 - return $product->ID;
3850 - }
3851 -
3852 - // Alternative method: query by post_name
3853 - global $wpdb;
3854 - $post_id = $wpdb->get_var($wpdb->prepare(
3855 - "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'product' AND post_status = 'publish'",
3856 - $product_slug
3857 - ));
3858 -
3859 - if ($post_id) {
3860 - return intval($post_id);
3861 - }
3862 - }
3863 - }
3864 -
3865 - // Generic approach: try to extract slug and match against all post types
3866 - $parsed_url = wp_parse_url($clean_url);
3867 - $path = $parsed_url['path'] ?? '';
3868 -
3869 - if (!empty($path)) {
3870 - // Get the last part of the path as potential slug
3871 - $path_parts = array_filter(explode('/', trim($path, '/')));
3872 - $potential_slug = end($path_parts);
3873 -
3874 - if (!empty($potential_slug)) {
3875 - global $wpdb;
3876 -
3877 - // Try to find any post with this slug
3878 - $post_id = $wpdb->get_var($wpdb->prepare(
3879 - "SELECT ID FROM {$wpdb->posts}
3880 - WHERE post_name = %s
3881 - AND post_status IN ('publish', 'closed', 'private')
3882 - AND post_type NOT IN ('revision', 'attachment', 'nav_menu_item')
3883 - ORDER BY CASE
3884 - WHEN post_type = 'post' THEN 1
3885 - WHEN post_type = 'page' THEN 2
3886 - WHEN post_type = 'topic' THEN 3
3887 - WHEN post_type = 'product' THEN 4
3888 - ELSE 5
3889 - END
3890 - LIMIT 1",
3891 - $potential_slug
3892 - ));
3893 -
3894 - if ($post_id) {
3895 - return intval($post_id);
3896 - }
3897 - }
3898 - }
3899 -
3900 - // ADDITIONAL: Try direct database lookup by URL variations
3901 - global $wpdb;
3902 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3903 -
3904 - // Try variations of the URL (with/without trailing slash, http/https)
3905 - $url_variations = array(
3906 - $url,
3907 - rtrim($url, '/'),
3908 - $url . '/',
3909 - str_replace('http://', 'https://', $url),
3910 - str_replace('https://', 'http://', $url),
3911 - str_replace('http://', 'https://', rtrim($url, '/')),
3912 - str_replace('https://', 'http://', rtrim($url, '/'))
3913 - );
3914 -
3915 - // Remove duplicates
3916 - $url_variations = array_unique($url_variations);
3917 -
3918 - foreach ($url_variations as $variation) {
3919 - $existing_record = $wpdb->get_row($wpdb->prepare(
3920 - "SELECT id, source_url FROM $table_name WHERE source_url = %s",
3921 - $variation
3922 - ));
3923 -
3924 - if ($existing_record) {
3925 - // Try to get post ID from this stored URL
3926 - $stored_post_id = url_to_postid($existing_record->source_url);
3927 - if ($stored_post_id > 0) {
3928 - return $stored_post_id;
3929 - }
3930 - }
3931 - }
3932 -
3933 - return 0; // No match found
3934 -}
3935 -/**
3936 - * Process selected content via AJAX
3937 - */
3938 1966 public function ajax_mxchat_process_selected_content() {
3939 1967 // Basic request validation
3940 1968 if (!check_ajax_referer('mxchat_content_selector_nonce', 'nonce', false)) {
3941 1969 wp_send_json_error('Invalid nonce');
@@ -3959,16 +1987,8 @@
3959 1987 wp_send_json_error('No content selected');
3960 1988 exit;
3961 1989 }
3962 1990
3963 - // Get bot_id from request
3964 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
3965 -
3966 - // ACF→PDF extraction is an install-level setting (Knowledge → ACF Fields,
3967 - // plan 11720c). The import modal shows a passive status line pointing
3968 - // there; the old per-batch checkbox and its remembered default are gone.
3969 - $extract_acf_pdfs = get_option('mxchat_acf_pdf_extraction', '0') === '1';
3970 -
3971 1991 // Process only ONE post at a time to avoid request size issues
3972 1992 $post_id = reset($post_ids);
3973 1993 $post = get_post($post_id);
3974 1994
@@ -3975,258 +1995,54 @@
3975 1995 if (!$post) {
3976 1996 wp_send_json_error('Post not found');
3977 1997 exit;
3978 1998 }
3979 -
3980 - /**
3981 - * Allow developers to modify post data before processing into the knowledge base.
3982 - * Applied on BOTH content-preparation paths (this manual bulk import and the
3983 - * auto-sync path in mxchat_handle_post_update) with the same signature, so a
3984 - * callback registered once covers every indexing route. Purely additive —
3985 - * zero behaviour change when unhooked.
3986 - *
3987 - * @param WP_Post $post The post about to be indexed.
3988 - * @param string $bot_id Bot context for this import.
3989 - */
3990 - $post = apply_filters('mxchat_before_process_post', $post, $bot_id);
3991 - if (!($post instanceof WP_Post)) {
3992 - $post = get_post($post_id); // defend against a bad callback return
1999 +
2000 + // Get minimal content
2001 + $content = $post->post_title . "\n\n" . wp_strip_all_tags($post->post_content);
2002 + $content = substr($content, 0, 10000); // Limit content size
2003 +
2004 + // Get API key with proper model detection
2005 + $options = get_option('mxchat_options');
2006 + $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
2007 +
2008 + if (strpos($selected_model, 'voyage') === 0) {
2009 + $api_key = $options['voyage_api_key'] ?? '';
2010 + $provider_name = 'Voyage AI';
2011 + } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
2012 + $api_key = $options['gemini_api_key'] ?? '';
2013 + $provider_name = 'Google Gemini';
2014 + } else {
2015 + $api_key = $options['api_key'] ?? '';
2016 + $provider_name = 'OpenAI';
3993 2017 }
3994 -
3995 - // Get content including title, short description (for WooCommerce), and main content
3996 - $content = $post->post_title . "\n\n";
3997 -
3998 - // Add short description if it exists (WooCommerce products use post_excerpt for short description)
3999 - // Strip FIRST, then test: an excerpt that is nothing but shortcodes strips to
4000 - // empty, and testing the raw value emitted a bare "Short Description: " label
4001 - // with no value after it. Matches mxchat_index_published_post.
4002 - // trim() only in the TEST — the emitted value is untouched, so a populated
4003 - // excerpt is byte-identical to before. A whitespace-only excerpt is an empty
4004 - // excerpt and must not produce a labelled line with nothing after it.
4005 - $clean_excerpt = $this->strip_shortcode_tags_preserve_content($post->post_excerpt);
4006 - if (trim($clean_excerpt) !== '') {
4007 - $content .= "Short Description: " . wp_strip_all_tags($clean_excerpt) . "\n\n";
4008 - }
4009 -
4010 - // Add main content - remove shortcode tags but preserve content inside them
4011 - $clean_content = $this->strip_shortcode_tags_preserve_content($post->post_content);
4012 - $content .= wp_strip_all_tags($clean_content);
4013 -
4014 - // ADD WOOCOMMERCE PRODUCT DATA (pricing, stock, categories, custom tabs)
4015 - if (get_post_type($post_id) === 'product' && class_exists('WooCommerce')) {
4016 - $product = wc_get_product($post_id);
4017 -
4018 - if ($product) {
4019 - // Get pricing information
4020 - $regular_price = $product->get_regular_price();
4021 - $sale_price = $product->get_sale_price();
4022 - $price = $product->get_price();
4023 - $sku = $product->get_sku();
4024 -
4025 - // Get currency symbol
4026 - $currency_symbol = get_woocommerce_currency_symbol();
4027 -
4028 - // Add pricing information
4029 - $content .= "\n";
4030 - if (!empty($regular_price)) {
4031 - $content .= "Price: " . $currency_symbol . $regular_price . "\n";
4032 - } elseif (!empty($price)) {
4033 - $content .= "Price: " . $currency_symbol . $price . "\n";
4034 - }
4035 -
4036 - if (!empty($sale_price) && $sale_price !== $regular_price) {
4037 - $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
4038 - }
4039 -
4040 - // Handle variable products - show price range
4041 - if ($product->is_type('variable')) {
4042 - $min_price = $product->get_variation_price('min');
4043 - $max_price = $product->get_variation_price('max');
4044 - if ($min_price !== $max_price) {
4045 - $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
4046 - }
4047 - }
4048 -
4049 - if (!empty($sku)) {
4050 - $content .= "SKU: " . $sku . "\n";
4051 - }
4052 -
4053 - // Get product categories
4054 - $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
4055 - if (!empty($categories) && !is_wp_error($categories)) {
4056 - $content .= "Categories: " . implode(', ', $categories) . "\n";
4057 - }
4058 - }
4059 -
4060 - // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
4061 - $custom_tabs = get_post_meta($post_id, 'yikes_woo_products_tabs', true);
4062 - if (!empty($custom_tabs) && is_array($custom_tabs)) {
4063 - foreach ($custom_tabs as $tab) {
4064 - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
4065 - $tab_content = isset($tab['content']) ? $tab['content'] : '';
4066 -
4067 - if (!empty($tab_title) && !empty($tab_content)) {
4068 - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
4069 - }
4070 - }
4071 - }
4072 -
4073 - // Also check for reusable/saved tabs applied to this product
4074 - $applied_saved_tabs = get_post_meta($post_id, 'yikes_woo_reusable_products_tabs_applied', true);
4075 - if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
4076 - $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
4077 - if (!empty($saved_tabs) && is_array($saved_tabs)) {
4078 - foreach ($applied_saved_tabs as $saved_tab_id) {
4079 - if (isset($saved_tabs[$saved_tab_id])) {
4080 - $tab = $saved_tabs[$saved_tab_id];
4081 - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
4082 - $tab_content = isset($tab['content']) ? $tab['content'] : '';
4083 -
4084 - if (!empty($tab_title) && !empty($tab_content)) {
4085 - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
4086 - }
4087 - }
4088 - }
4089 - }
4090 - }
4091 - }
4092 -
4093 - // For custom post types like job_listing, include additional fields
4094 - // (verbatim parity with mxchat_index_published_post — a bulk import used to
4095 - // index the body alone, losing location/type/company that auto-sync captured)
4096 - if (get_post_type($post_id) === 'job_listing') {
4097 - // Add job-specific meta if available
4098 - $job_location = get_post_meta($post_id, '_job_location', true);
4099 - if (!empty($job_location)) {
4100 - $content .= "\n\nLocation: " . $job_location;
4101 - }
4102 -
4103 - // Get job type terms
4104 - $job_types = get_the_terms($post_id, 'job_listing_type');
4105 - if (!empty($job_types) && !is_wp_error($job_types)) {
4106 - $types = array();
4107 - foreach ($job_types as $type) {
4108 - $types[] = $type->name;
4109 - }
4110 - $content .= "\n\nJob Type: " . implode(', ', $types);
4111 - }
4112 -
4113 - // Get company name if available
4114 - $company_name = get_post_meta($post_id, '_company_name', true);
4115 - if (!empty($company_name)) {
4116 - $content .= "\n\nCompany: " . $company_name;
4117 - }
4118 - }
4119 -
4120 - // ADD ACF FIELDS SUPPORT
4121 - $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
4122 - $pdf_extracted_count = 0;
4123 - if (!empty($acf_fields)) {
4124 - $acf_content_parts = array();
4125 - $pdf_attachment_ids = array();
4126 -
4127 - foreach ($acf_fields as $field_name => $field_value) {
4128 - $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
4129 -
4130 - if (!empty($formatted_value)) {
4131 - // Both separators: a hyphenated ACF name should read as words, and
4132 - // this is what mxchat_index_published_post already does.
4133 - $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name));
4134 - $acf_content_parts[] = $field_label . ": " . $formatted_value;
4135 - }
4136 -
4137 - // Walk this field's value tree for any PDF attachment references and queue them for extraction.
4138 - // Only when the user opted into ACF→PDF extraction for this batch; otherwise the ACF text
4139 - // still lands in the KB but the heavier PDF parsing is skipped.
4140 - if ($extract_acf_pdfs) {
4141 - $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
4142 - }
4143 - }
4144 -
4145 - if (!empty($acf_content_parts)) {
4146 - $content .= "\n\n" . implode("\n", $acf_content_parts);
4147 - }
4148 -
4149 - // Extract text from each unique PDF found in ACF fields and append as a labeled section
4150 - if ($extract_acf_pdfs && !empty($pdf_attachment_ids)) {
4151 - $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
4152 - $pdf_sections = array();
4153 - foreach ($pdf_attachment_ids as $att_id) {
4154 - $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
4155 - if (!empty($pdf_text)) {
4156 - $pdf_title = get_the_title($att_id);
4157 - $pdf_url = wp_get_attachment_url($att_id);
4158 - $header = 'PDF Attachment';
4159 - if (!empty($pdf_title)) {
4160 - $header .= ': ' . $pdf_title;
4161 - }
4162 - if (!empty($pdf_url)) {
4163 - $header .= ' (' . $pdf_url . ')';
4164 - }
4165 - $pdf_sections[] = $header . "\n" . $pdf_text;
4166 - $pdf_extracted_count++;
4167 - }
4168 - }
4169 - if (!empty($pdf_sections)) {
4170 - $content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
4171 - }
4172 - }
4173 - }
4174 -
4175 - // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
4176 - $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
4177 - if (!empty($custom_meta)) {
4178 - $meta_content_parts = array();
4179 -
4180 - foreach ($custom_meta as $meta_key => $meta_value) {
4181 - // Convert meta key to readable label
4182 - $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
4183 - $meta_content_parts[] = $meta_label . ": " . $meta_value;
4184 - }
4185 -
4186 - if (!empty($meta_content_parts)) {
4187 - $content .= "\n\n" . implode("\n", $meta_content_parts);
4188 - }
4189 - }
4190 -
4191 - // Debug logging for WordPress Import content
4192 - //error_log('[MXCHAT-WP-IMPORT-DEBUG] Post ID: ' . $post_id . ' Title: ' . $post->post_title);
4193 - //error_log('[MXCHAT-WP-IMPORT-DEBUG] Raw post_content length: ' . strlen($post->post_content));
4194 - //error_log('[MXCHAT-WP-IMPORT-DEBUG] Final content length: ' . strlen($content));
4195 - //error_log('[MXCHAT-WP-IMPORT-DEBUG] Content preview: ' . substr($content, 0, 300));
4196 -
4197 - // Note: Removed 10,000 char limit - chunking now handles large content properly
4198 -
4199 - // Get bot-specific embedding decision — custom-provider-aware (plan cbd5fd)
4200 - $bot_options = $this->get_bot_options($bot_id);
4201 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
4202 -
4203 - $preflight = MxChat_Utils::embedding_preflight($options);
4204 - if (!$preflight['ok']) {
4205 - MxChat_Admin::mxchat_log_debug('api_error', $preflight['reason'] . ' (knowledge processing)');
4206 - wp_send_json_error($preflight['reason']);
2018 +
2019 + if (empty($api_key)) {
2020 + wp_send_json_error($provider_name . ' API key not configured');
4207 2021 exit;
4208 2022 }
4209 - $api_key = $preflight['api_key'];
4210 2023
4211 2024 $source_url = get_permalink($post_id);
4212 2025 $vector_id = md5($source_url); // Vector ID for Pinecone
4213 2026
4214 - // Check for existing content in bot-specific storage
2027 + // ================================
2028 + // FIXED: Check for existing content in ONLY the active storage method
2029 + // ================================
2030 +
4215 2031 $is_update = false;
4216 2032
4217 - // Get bot-specific Pinecone configuration
4218 - $bot_pinecone_config = $this->get_bot_pinecone_config($bot_id);
4219 - $use_pinecone = !empty($bot_pinecone_config) && ($bot_pinecone_config['use_pinecone'] ?? false);
2033 + // Check if Pinecone is enabled
2034 + $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
2035 + $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
4220 2036
4221 - if ($use_pinecone && !empty($bot_pinecone_config['api_key'])) {
4222 - // Check Pinecone for this bot
4223 - $pinecone_data = $this->mxchat_get_pinecone_processed_content($bot_pinecone_config);
2037 + if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
2038 + // ONLY check Pinecone if it's enabled
2039 + $pinecone_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
4224 2040 if (isset($pinecone_data[$post_id])) {
4225 2041 $is_update = true;
4226 2042 }
4227 2043 } else {
4228 - // Check WordPress DB (same as before since it's shared)
2044 + // ONLY check WordPress DB if Pinecone is not enabled
4229 2045 global $wpdb;
4230 2046 $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
4231 2047 $existing_record = $wpdb->get_row($wpdb->prepare(
4232 2048 "SELECT id FROM $table_name WHERE source_url = %s",
@@ -4236,54 +2052,54 @@
4236 2052 if ($existing_record) {
4237 2053 $is_update = true;
4238 2054 }
4239 2055 }
4240 -
4241 - // UPDATED 2.5.6: Determine content type based on post_type
4242 - $post_type = $post->post_type;
4243 - $content_type = 'content'; // Default fallback
4244 -
4245 - // Map WordPress post types to content types
4246 - switch ($post_type) {
4247 - case 'post':
4248 - $content_type = 'post';
4249 - break;
4250 - case 'page':
4251 - $content_type = 'page';
4252 - break;
4253 - case 'product':
4254 - $content_type = 'product';
4255 - break;
4256 - default:
4257 - // For custom post types, use the post type name
4258 - $content_type = sanitize_key($post_type);
4259 - break;
4260 - }
4261 -
4262 - // Use the centralized utility function with bot_id and content_type
2056 +
2057 + // Use the centralized utility function for storage
4263 2058 $result = MxChat_Utils::submit_content_to_db(
4264 - $content,
4265 - $source_url,
2059 + $content,
2060 + $source_url,
4266 2061 $api_key,
4267 - $vector_id,
4268 - $bot_id,
4269 - $content_type
2062 + $vector_id
4270 2063 );
4271 -
2064 +
4272 2065 if (is_wp_error($result)) {
4273 - MxChat_Admin::mxchat_log_debug('storage_error', 'Knowledge storage failed: ' . $result->get_error_message(), array('source_url' => $source_url));
4274 2066 wp_send_json_error('Storage failed: ' . $result->get_error_message());
4275 2067 exit;
4276 2068 }
4277 -
4278 - // Automatically apply role restriction based on tags
4279 - $this->apply_role_restriction_to_post($post_id, $source_url);
4280 -
2069 +
2070 + // ================================
2071 + // UPDATE: Only update caches if Pinecone is enabled
2072 + // ================================
2073 +
2074 + if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
2075 + // Update vector ID cache for improved fetching
2076 + $this->mxchat_update_pinecone_vector_cache($vector_id);
2077 +
2078 + // Update local processed content cache for immediate UI feedback
2079 + $pinecone_cache = get_option('mxchat_pinecone_processed_cache', array());
2080 + $pinecone_cache[$post_id] = array(
2081 + 'db_id' => $vector_id,
2082 + 'processed_date' => 'Just now',
2083 + 'url' => $source_url,
2084 + 'source' => 'pinecone',
2085 + 'timestamp' => current_time('timestamp')
2086 + );
2087 + update_option('mxchat_pinecone_processed_cache', $pinecone_cache);
2088 +
2089 + // Also update the general processed content cache
2090 + $processed_cache = get_option('mxchat_processed_content_cache', array());
2091 + $processed_cache[$post_id] = array(
2092 + 'db_id' => $vector_id,
2093 + 'timestamp' => current_time('timestamp'),
2094 + 'url' => $source_url,
2095 + 'source' => 'pinecone'
2096 + );
2097 + update_option('mxchat_processed_content_cache', $processed_cache);
2098 + }
2099 +
4281 2100 $operation_type = $is_update ? 'update' : 'new';
4282 2101
4283 - // Count ACF fields for debugging
4284 - $acf_field_count = count($acf_fields);
4285 -
4286 2102 // Success response with minimal data
4287 2103 wp_send_json_success(array(
4288 2104 'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
4289 2105 'post_id' => $post_id,
@@ -4288,95 +2104,27 @@
4288 2104 'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
4289 2105 'post_id' => $post_id,
4290 2106 'title' => $post->post_title,
4291 2107 'operation_type' => $operation_type,
4292 - 'vector_id' => $vector_id,
4293 - 'acf_fields_found' => $acf_field_count,
4294 - 'pdf_extracted_count' => (int) $pdf_extracted_count,
4295 - 'content_preview' => substr($content, 0, 100) . '...',
4296 - 'bot_id' => $bot_id
2108 + 'vector_id' => $vector_id, // Include vector ID for debugging
2109 + 'cache_updated' => $use_pinecone // Indicate if cache was updated
4297 2110 ));
4298 2111 exit;
4299 2112 }
4300 2113
4301 -private function apply_role_restriction_to_post($post_id, $source_url) {
4302 - // Get tag-role mappings
4303 - $mappings = get_option('mxchat_tag_role_mappings', array());
4304 -
4305 - if (empty($mappings)) {
4306 - return; // No mappings, leave as public
4307 - }
4308 -
4309 - // Get all tags for the post
4310 - $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
4311 -
4312 - if (empty($post_tags)) {
4313 - return; // No tags, leave as public
4314 - }
4315 -
4316 - // Determine the highest role restriction based on tags
4317 - $highest_role = 'public';
4318 - $role_hierarchy = array(
4319 - 'public' => 0,
4320 - 'logged_in' => 1,
4321 - 'subscriber' => 2,
4322 - 'contributor' => 3,
4323 - 'author' => 4,
4324 - 'editor' => 5,
4325 - 'administrator' => 6
4326 - );
4327 -
4328 - foreach ($post_tags as $tag_slug) {
4329 - if (isset($mappings[$tag_slug])) {
4330 - $role = $mappings[$tag_slug];
4331 - if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
4332 - $highest_role = $role;
4333 - }
4334 - }
4335 - }
4336 -
4337 - // If no restricted tags found, return (leave as public)
4338 - if ($highest_role === 'public') {
4339 - return;
4340 - }
4341 -
4342 - // Update the role restriction in the database
4343 - global $wpdb;
4344 -
4345 - // Check if using Pinecone
4346 - $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
4347 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
4348 -
4349 - if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
4350 - // Update Pinecone role restriction
4351 - $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
4352 - $vector_id = md5($source_url);
4353 -
4354 - $wpdb->replace(
4355 - $roles_table,
4356 - array(
4357 - 'vector_id' => $vector_id,
4358 - 'role_restriction' => $highest_role,
4359 - 'updated_at' => current_time('mysql')
4360 - ),
4361 - array('%s', '%s', '%s')
4362 - );
4363 - } else {
4364 - // Update WordPress DB
4365 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
4366 -
4367 - $wpdb->update(
4368 - $table_name,
4369 - array('role_restriction' => $highest_role),
4370 - array('source_url' => $source_url),
4371 - array('%s'),
4372 - array('%s')
4373 - );
4374 - }
4375 -}
4376 2114
2115 +
2116 + /**
2117 + * Updates cache with new vector ID if absent
2118 + */
2119 + public function mxchat_update_pinecone_vector_cache($vector_id) {
2120 + $cached_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
2121 + if (!in_array($vector_id, $cached_ids)) {
2122 + $cached_ids[] = $vector_id;
2123 + update_option('mxchat_pinecone_vector_ids_cache', $cached_ids);
2124 + }
2125 + }
4377 2126 public function mxchat_get_public_post_types() {
4378 - // Get all public post types
4379 2127 $post_types = get_post_types(array('public' => true), 'objects');
4380 2128 $post_type_options = array();
4381 2129
4382 2130 foreach ($post_types as $post_type) {
@@ -4382,45 +2130,51 @@
4382 2130 foreach ($post_types as $post_type) {
4383 2131 $post_type_options[$post_type->name] = $post_type->label;
4384 2132 }
4385 2133
4386 - // Also include common forum/community post types that might not be marked as public
4387 - $additional_types = array(
4388 - 'topic' => 'Forum Topics (bbPress)',
4389 - 'reply' => 'Forum Replies (bbPress)',
4390 - 'forum' => 'Forums (bbPress)',
4391 - 'wpforo_topic' => 'wpForo Topics',
4392 - 'wpforo_post' => 'wpForo Posts'
4393 - );
4394 -
4395 - foreach ($additional_types as $type_name => $type_label) {
4396 - if (post_type_exists($type_name) && !isset($post_type_options[$type_name])) {
4397 - $post_type_options[$type_name] = $type_label;
4398 - }
4399 - }
4400 -
4401 2134 return $post_type_options;
4402 2135 }
2136 +public function mxchat_get_pinecone_processed_content($pinecone_options) {
2137 + //error_log('=== DEBUG: Starting mxchat_get_pinecone_processed_content ===');
2138 +
2139 + // First check local cache for immediate updates
2140 + $cached_data = get_option('mxchat_pinecone_processed_cache', array());
2141 + //error_log('DEBUG: Found ' . count($cached_data) . ' items in local cache');
4403 2142
4404 -/**
4405 - * Retrieves processed content from Pinecone API
4406 - */
4407 -public function mxchat_get_pinecone_processed_content($pinecone_options) {
4408 2143 $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4409 2144 $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
2145 +
2146 + //error_log('DEBUG: API key present: ' . (!empty($api_key) ? 'YES' : 'NO'));
2147 + //error_log('DEBUG: Host: ' . $host);
4410 2148
4411 2149 if (empty($api_key) || empty($host)) {
4412 - return array();
2150 + //error_log('DEBUG: Missing API credentials, returning cached data only');
2151 + return $cached_data;
4413 2152 }
4414 2153
4415 2154 $pinecone_data = array();
4416 2155
4417 2156 try {
4418 - // Always get fresh data from Pinecone
4419 - $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
2157 + // Method 1: Try to get vectors using cached vector IDs first
2158 + $cached_vector_ids = get_option('mxchat_pinecone_vector_ids_cache', array());
2159 + //error_log('DEBUG: Found ' . count($cached_vector_ids) . ' cached vector IDs');
4420 2160
4421 - // Method 2: Final fallback - try stats endpoint (if available)
2161 + if (!empty($cached_vector_ids)) {
2162 + //error_log('DEBUG: Trying to fetch by cached vector IDs...');
2163 + $pinecone_data = $this->mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $cached_vector_ids);
2164 + //error_log('DEBUG: Fetch by IDs returned ' . count($pinecone_data) . ' items');
2165 + }
2166 +
2167 + // Method 2: If no cached IDs or fetch failed, use scanning approach
4422 2168 if (empty($pinecone_data)) {
2169 + //error_log('DEBUG: Trying scanning approach...');
2170 + $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);
2171 + //error_log('DEBUG: Scanning returned ' . count($pinecone_data) . ' items');
2172 + }
2173 +
2174 + // Method 3: Final fallback - try stats endpoint
2175 + if (empty($pinecone_data)) {
2176 + //error_log('DEBUG: Trying stats endpoint...');
4423 2177 $stats_url = "https://{$host}/describe_index_stats";
4424 2178
4425 2179 $response = wp_remote_post($stats_url, array(
4426 2180 'headers' => array(
@@ -4433,17 +2187,44 @@
4433 2187
4434 2188 if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
4435 2189 $body = wp_remote_retrieve_body($response);
4436 2190 $stats_data = json_decode($body, true);
2191 + //error_log('DEBUG: Pinecone stats: ' . print_r($stats_data, true));
2192 + } else {
2193 + if (is_wp_error($response)) {
2194 + //error_log('DEBUG: Stats endpoint error: ' . $response->get_error_message());
2195 + } else {
2196 + //error_log('DEBUG: Stats endpoint failed with code: ' . wp_remote_retrieve_response_code($response));
2197 + }
4437 2198 }
4438 2199 }
4439 2200
4440 2201 } catch (Exception $e) {
4441 - // Log error but return fresh data only
2202 + //error_log('DEBUG: Exception in get_pinecone_processed_content: ' . $e->getMessage());
4442 2203 }
4443 2204
4444 - return $pinecone_data;
2205 + // Merge cached data with Pinecone data
2206 + $merged_data = $pinecone_data;
2207 +
2208 + foreach ($cached_data as $post_id => $cache_item) {
2209 + $cache_timestamp = $cache_item['timestamp'] ?? 0;
2210 + $time_diff = current_time('timestamp') - $cache_timestamp;
2211 +
2212 + if ($time_diff < 300) { // 5 minutes = 300 seconds
2213 + $merged_data[$post_id] = $cache_item;
2214 + } else {
2215 + if (!isset($merged_data[$post_id])) {
2216 + $merged_data[$post_id] = $cache_item;
2217 + }
2218 + }
2219 + }
2220 +
2221 + //error_log('DEBUG: Final merged data count: ' . count($merged_data));
2222 + //error_log('=== DEBUG: End mxchat_get_pinecone_processed_content ===');
2223 +
2224 + return $merged_data;
4445 2225 }
2226 +
4446 2227 public function mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $vector_ids) {
4447 2228 //error_log('=== DEBUG: Starting mxchat_fetch_pinecone_vectors_by_ids ===');
4448 2229
4449 2230 $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
@@ -4535,46 +2316,16 @@
4535 2316 return array();
4536 2317 }
4537 2318 }
4538 2319
4539 -/**
4540 - * Get embedding dimensions based on the selected model.
4541 - */
4542 -private function mxchat_get_embedding_dimensions() {
4543 - $options = get_option('mxchat_options', array());
4544 - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4545 -
4546 - $model_dimensions = array(
4547 - 'text-embedding-ada-002' => 1536,
4548 - 'text-embedding-3-small' => 1536,
4549 - 'text-embedding-3-large' => 3072,
4550 - 'voyage-2' => 1024,
4551 - 'voyage-large-2' => 1536,
4552 - 'voyage-3-large' => 2048,
4553 - 'gemini-embedding-001' => 1536,
4554 - );
4555 -
4556 - if (strpos($selected_model, 'voyage-3-large') === 0) {
4557 - $custom_dimensions = $options['voyage_output_dimension'] ?? 2048;
4558 - return intval($custom_dimensions);
4559 - }
4560 -
4561 - if (strpos($selected_model, 'gemini-embedding') === 0) {
4562 - $custom_dimensions = $options['gemini_output_dimension'] ?? 1536;
4563 - return intval($custom_dimensions);
4564 - }
4565 -
4566 - return $model_dimensions[$selected_model] ?? 1536;
4567 -}
4568 -
4569 -/**
4570 - * Scan Pinecone for processed content
4571 - */
4572 2320 public function mxchat_scan_pinecone_for_processed_content($pinecone_options) {
2321 + //error_log('=== DEBUG: Starting mxchat_scan_pinecone_for_processed_content ===');
2322 +
4573 2323 $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
4574 2324 $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
4575 2325
4576 2326 if (empty($api_key) || empty($host)) {
2327 + //error_log('DEBUG: Missing API credentials for scanning');
4577 2328 return array();
4578 2329 }
4579 2330
4580 2331 try {
@@ -4581,18 +2332,17 @@
4581 2332 // Use multiple random vectors to get better coverage
4582 2333 $all_matches = array();
4583 2334 $seen_ids = array();
4584 2335
4585 - // Get correct dimensions for the configured embedding model
4586 - $dimensions = $this->mxchat_get_embedding_dimensions();
4587 -
4588 2336 // Try 3 different random vectors to get better coverage
4589 2337 for ($i = 0; $i < 3; $i++) {
2338 + //error_log('DEBUG: Scanning attempt ' . ($i + 1) . '/3');
2339 +
4590 2340 $query_url = "https://{$host}/query";
4591 2341
4592 2342 // Generate a random unit vector instead of zeros
4593 2343 $random_vector = array();
4594 - for ($j = 0; $j < $dimensions; $j++) {
2344 + for ($j = 0; $j < 1536; $j++) {
4595 2345 $random_vector[] = (rand(-1000, 1000) / 1000.0);
4596 2346 }
4597 2347
4598 2348 // Normalize the vector to unit length
@@ -4617,14 +2367,18 @@
4617 2367 'timeout' => 30
4618 2368 ));
4619 2369
4620 2370 if (is_wp_error($response)) {
2371 + //error_log('DEBUG: Query attempt ' . ($i + 1) . ' WP error: ' . $response->get_error_message());
4621 2372 continue;
4622 2373 }
4623 2374
4624 2375 $response_code = wp_remote_retrieve_response_code($response);
2376 + //error_log('DEBUG: Query attempt ' . ($i + 1) . ' response code: ' . $response_code);
4625 2377
4626 2378 if ($response_code !== 200) {
2379 + $error_body = wp_remote_retrieve_body($response);
2380 + //error_log('DEBUG: Query attempt ' . ($i + 1) . ' failed with body: ' . substr($error_body, 0, 500));
4627 2381 continue;
4628 2382 }
4629 2383
4630 2384 $body = wp_remote_retrieve_body($response);
@@ -4630,8 +2384,9 @@
4630 2384 $body = wp_remote_retrieve_body($response);
4631 2385 $data = json_decode($body, true);
4632 2386
4633 2387 if (isset($data['matches'])) {
2388 + //error_log('DEBUG: Query attempt ' . ($i + 1) . ' returned ' . count($data['matches']) . ' matches');
4634 2389 foreach ($data['matches'] as $match) {
4635 2390 $match_id = $match['id'] ?? '';
4636 2391 if (!empty($match_id) && !isset($seen_ids[$match_id])) {
4637 2392 $all_matches[] = $match;
@@ -4637,14 +2392,18 @@
4637 2392 $all_matches[] = $match;
4638 2393 $seen_ids[$match_id] = true;
4639 2394 }
4640 2395 }
2396 + } else {
2397 + //error_log('DEBUG: Query attempt ' . ($i + 1) . ' - no matches key in response');
4641 2398 }
4642 2399 }
4643 2400
4644 - // Convert matches to processed data format, grouping by URL to count chunks
2401 + //error_log('DEBUG: Total unique matches found: ' . count($all_matches));
2402 +
2403 + // Convert matches to processed data format
4645 2404 $processed_data = array();
4646 - $url_chunk_counts = array();
2405 + $vector_ids_for_cache = array();
4647 2406
4648 2407 foreach ($all_matches as $match) {
4649 2408 $metadata = $match['metadata'] ?? array();
4650 2409 $source_url = $metadata['source_url'] ?? '';
@@ -4652,14 +2411,8 @@
4652 2411
4653 2412 if (!empty($source_url) && !empty($match_id)) {
4654 2413 $post_id = url_to_postid($source_url);
4655 2414 if ($post_id) {
4656 - // Count chunks per post_id
4657 - if (!isset($url_chunk_counts[$post_id])) {
4658 - $url_chunk_counts[$post_id] = 0;
4659 - }
4660 - $url_chunk_counts[$post_id]++;
4661 -
4662 2415 $created_at = $metadata['created_at'] ?? '';
4663 2416 $processed_date = 'Recently';
4664 2417
4665 2418 if (!empty($created_at)) {
@@ -4668,332 +2421,243 @@
4668 2421 $processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
4669 2422 }
4670 2423 }
4671 2424
4672 - // Only store if not already set, or update with newer timestamp
4673 - if (!isset($processed_data[$post_id]) ||
4674 - ($timestamp ?? 0) > ($processed_data[$post_id]['timestamp'] ?? 0)) {
4675 - $processed_data[$post_id] = array(
4676 - 'db_id' => $match_id,
4677 - 'processed_date' => $processed_date,
4678 - 'url' => $source_url,
4679 - 'source' => 'pinecone',
4680 - 'timestamp' => $timestamp ?? current_time('timestamp')
4681 - );
4682 - }
2425 + $processed_data[$post_id] = array(
2426 + 'db_id' => $match_id,
2427 + 'processed_date' => $processed_date,
2428 + 'url' => $source_url,
2429 + 'source' => 'pinecone',
2430 + 'timestamp' => $timestamp ?? current_time('timestamp')
2431 + );
2432 +
2433 + $vector_ids_for_cache[] = $match_id;
4683 2434 }
4684 2435 }
4685 2436 }
4686 2437
4687 - // Add chunk counts to processed data
4688 - foreach ($url_chunk_counts as $post_id => $chunk_count) {
4689 - if (isset($processed_data[$post_id])) {
4690 - $processed_data[$post_id]['chunk_count'] = $chunk_count;
4691 - }
2438 + // Update the vector IDs cache for future use
2439 + if (!empty($vector_ids_for_cache)) {
2440 + update_option('mxchat_pinecone_vector_ids_cache', $vector_ids_for_cache);
2441 + //error_log('DEBUG: Updated vector IDs cache with ' . count($vector_ids_for_cache) . ' IDs');
4692 2442 }
4693 2443
2444 + //error_log('DEBUG: Returning ' . count($processed_data) . ' processed items from scanning');
4694 2445 return $processed_data;
4695 2446
4696 2447 } catch (Exception $e) {
2448 + //error_log('DEBUG: Exception in scan_pinecone_for_processed_content: ' . $e->getMessage());
4697 2449 return array();
4698 2450 }
4699 2451 }
4700 -/**
4701 - * Generate embeddings from input text for MXChat with bot support
4702 - */
4703 -private function mxchat_generate_embedding($text, $bot_id = 'default') {
4704 - // Enable detailed logging for debugging
4705 - //error_log('[MXCHAT-EMBED] Starting embedding generation for bot: ' . $bot_id . '. Text length: ' . strlen($text) . ' bytes');
4706 - //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4707 2452
4708 - // Get bot-specific options
4709 - $bot_options = $this->get_bot_options($bot_id);
4710 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
2453 + /**
2454 + * Generates embeddings from input text for MXChat
2455 + */
2456 + private function mxchat_generate_embedding($text) {
2457 + // Enable detailed logging for debugging
2458 + //error_log('[MXCHAT-EMBED] Starting embedding generation. Text length: ' . strlen($text) . ' bytes');
2459 + //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');
4711 2460
4712 - // Opt-in: when the custom provider is selected for embeddings, index through
4713 - // the same custom endpoint the query path uses so stored vectors and query
4714 - // vectors share a model. Returns the vector array on success, or an error
4715 - // string on failure (this function's existing failure contract).
4716 - if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
4717 - if (!class_exists('MxChat_Utils')) {
4718 - require_once dirname(__FILE__) . '/../includes/class-mxchat-utils.php';
4719 - }
4720 - return MxChat_Utils::generate_embedding_custom($text, $options);
4721 - }
2461 + $options = get_option('mxchat_options');
2462 + $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
2463 + //error_log('[MXCHAT-EMBED] Selected embedding model: ' . $selected_model);
4722 2464
4723 - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
4724 - //error_log('[MXCHAT-EMBED] Selected embedding model for bot ' . $bot_id . ': ' . $selected_model);
2465 + // Determine provider and endpoint
2466 + if (strpos($selected_model, 'voyage') === 0) {
2467 + $api_key = $options['voyage_api_key'] ?? '';
2468 + $endpoint = 'https://api.voyageai.com/v1/embeddings';
2469 + $provider_name = 'Voyage AI';
2470 + //error_log('[MXCHAT-EMBED] Using Voyage AI API');
2471 + } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
2472 + $api_key = $options['gemini_api_key'] ?? '';
2473 + $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
2474 + $provider_name = 'Google Gemini';
2475 + //error_log('[MXCHAT-EMBED] Using Google Gemini API');
2476 + } else {
2477 + $api_key = $options['api_key'] ?? '';
2478 + $endpoint = 'https://api.openai.com/v1/embeddings';
2479 + $provider_name = 'OpenAI';
2480 + //error_log('[MXCHAT-EMBED] Using OpenAI API');
2481 + }
4725 2482
4726 - // Determine provider and endpoint
4727 - if (strpos($selected_model, 'voyage') === 0) {
4728 - $api_key = $options['voyage_api_key'] ?? '';
4729 - $endpoint = 'https://api.voyageai.com/v1/embeddings';
4730 - $provider_name = 'Voyage AI';
4731 - //error_log('[MXCHAT-EMBED] Using Voyage AI API for bot ' . $bot_id);
4732 - } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
4733 - $api_key = $options['gemini_api_key'] ?? '';
4734 - $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
4735 - $provider_name = 'Google Gemini';
4736 - //error_log('[MXCHAT-EMBED] Using Google Gemini API for bot ' . $bot_id);
4737 - } else {
4738 - $api_key = $options['api_key'] ?? '';
4739 - $endpoint = 'https://api.openai.com/v1/embeddings';
4740 - $provider_name = 'OpenAI';
4741 - //error_log('[MXCHAT-EMBED] Using OpenAI API for bot ' . $bot_id);
4742 - }
2483 + //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
4743 2484
4744 - //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);
2485 + if (empty($api_key)) {
2486 + $error_message = sprintf('Missing %s API key. Please configure your API key in the MxChat settings.', $provider_name);
2487 + //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
2488 + return $error_message;
2489 + }
4745 2490
4746 - if (empty($api_key)) {
4747 - $error_message = sprintf('Missing %s API key for bot %s. Please configure your API key in the bot settings.', $provider_name, $bot_id);
4748 - //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
4749 - return $error_message;
4750 - }
2491 + // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
2492 + $estimated_tokens = ceil(str_word_count($text) / 0.75);
2493 + //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
4751 2494
4752 - // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
4753 - $estimated_tokens = ceil(str_word_count($text) / 0.75);
4754 - //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);
2495 + if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
2496 + //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
2497 + // Consider truncating text here
2498 + }
4755 2499
4756 - if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
4757 - //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
4758 - // Consider truncating text here
4759 - }
2500 + // Prepare request body based on provider
2501 + if (strpos($selected_model, 'gemini-embedding') === 0) {
2502 + // Gemini API format
2503 + $request_body = array(
2504 + 'model' => 'models/' . $selected_model,
2505 + 'content' => array(
2506 + 'parts' => array(
2507 + array('text' => $text)
2508 + )
2509 + )
2510 + );
4760 2511
4761 - // Prepare request body based on provider
4762 - if (strpos($selected_model, 'gemini-embedding') === 0) {
4763 - // Gemini API format
4764 - $request_body = array(
4765 - 'model' => 'models/' . $selected_model,
4766 - 'content' => array(
4767 - 'parts' => array(
4768 - array('text' => $text)
4769 - )
4770 - )
4771 - );
2512 + // Set output dimensionality to 1536 for consistency with other models
2513 + $request_body['outputDimensionality'] = 1536;
2514 + } else {
2515 + // OpenAI/Voyage API format
2516 + $request_body = array(
2517 + 'model' => $selected_model,
2518 + 'input' => $text
2519 + );
4772 2520
4773 - // Set output dimensionality to 1536 for consistency with other models
4774 - $request_body['outputDimensionality'] = 1536;
4775 - } else {
4776 - // OpenAI/Voyage API format
4777 - $request_body = array(
4778 - 'model' => $selected_model,
4779 - 'input' => $text
4780 - );
2521 + // Add output_dimension for voyage-3-large model
2522 + if ($selected_model === 'voyage-3-large') {
2523 + $request_body['output_dimension'] = 2048;
2524 + }
2525 + }
4781 2526
4782 - // Add output_dimension for voyage-3-large model
4783 - if ($selected_model === 'voyage-3-large') {
4784 - $request_body['output_dimension'] = 2048;
4785 - }
4786 - }
2527 + //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
4787 2528
4788 - //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);
2529 + // Prepare headers based on provider
2530 + if (strpos($selected_model, 'gemini-embedding') === 0) {
2531 + // Gemini uses API key as query parameter
2532 + $endpoint .= '?key=' . $api_key;
2533 + $headers = array(
2534 + 'Content-Type' => 'application/json'
2535 + );
2536 + } else {
2537 + // OpenAI/Voyage use Bearer token
2538 + $headers = array(
2539 + 'Authorization' => 'Bearer ' . $api_key,
2540 + 'Content-Type' => 'application/json'
2541 + );
2542 + }
4789 2543
4790 - // Prepare headers based on provider
4791 - if (strpos($selected_model, 'gemini-embedding') === 0) {
4792 - // Gemini uses API key as query parameter
4793 - $endpoint .= '?key=' . $api_key;
4794 - $headers = array(
4795 - 'Content-Type' => 'application/json'
4796 - );
4797 - } else {
4798 - // OpenAI/Voyage use Bearer token
4799 - $headers = array(
4800 - 'Authorization' => 'Bearer ' . $api_key,
4801 - 'Content-Type' => 'application/json'
4802 - );
4803 - }
2544 + // Make API request
2545 + //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
2546 + $response = wp_remote_post($endpoint, array(
2547 + 'body' => wp_json_encode($request_body),
2548 + 'headers' => $headers,
2549 + 'timeout' => 60 // Increased timeout for large inputs
2550 + ));
4804 2551
4805 - // Make API request
4806 - //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
4807 - $response = wp_remote_post($endpoint, array(
4808 - 'body' => wp_json_encode($request_body),
4809 - 'headers' => $headers,
4810 - 'timeout' => 60 // Increased timeout for large inputs
4811 - ));
2552 + // Handle wp_remote_post errors
2553 + if (is_wp_error($response)) {
2554 + $error_message = $response->get_error_message();
2555 + //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
2556 + return 'Connection error: ' . $error_message;
2557 + }
4812 2558
4813 - // Handle wp_remote_post errors
4814 - if (is_wp_error($response)) {
4815 - $error_message = $response->get_error_message();
4816 - //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
4817 - return 'Connection error: ' . $error_message;
4818 - }
2559 + // Get and check HTTP response code
2560 + $http_code = wp_remote_retrieve_response_code($response);
2561 + //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
4819 2562
4820 - // Get and check HTTP response code
4821 - $http_code = wp_remote_retrieve_response_code($response);
4822 - //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);
2563 + if ($http_code !== 200) {
2564 + $error_body = wp_remote_retrieve_body($response);
2565 + //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
4823 2566
4824 - if ($http_code !== 200) {
4825 - $error_body = wp_remote_retrieve_body($response);
4826 - //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);
2567 + // Try to parse error for more details
2568 + $error_json = json_decode($error_body, true);
2569 + if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
2570 + $error_type = $error_json['error']['type'] ?? 'unknown';
2571 + $error_message = $error_json['error']['message'] ?? 'No message';
2572 + //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
2573 + //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
4827 2574
4828 - // Try to parse error for more details
4829 - $error_json = json_decode($error_body, true);
4830 - if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
4831 - $error_type = $error_json['error']['type'] ?? 'unknown';
4832 - $error_message = $error_json['error']['message'] ?? 'No message';
4833 - //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
4834 - //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);
2575 + // Customize error message for common API errors
2576 + if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) {
2577 + $error_message = sprintf('Invalid %s API key. Please check your API key in the MxChat settings.', $provider_name);
2578 + } elseif ($error_type === 'authentication_error') {
2579 + $error_message = sprintf('%s authentication failed. Please verify your API key in the MxChat settings.', $provider_name);
2580 + }
4835 2581
4836 - // Keep the provider's own diagnostic — a restricted-key 401 names the
4837 - // exact missing scope, and replacing it with "check your API key" sent
4838 - // a customer to regenerate two keys (plan 46b596). Same shape as
4839 - // MxChat_Utils::embedding_failure_error() so both ingestion paths read
4840 - // identically. Key never appears in provider messages, but scrub anyway.
4841 - if ($error_type === 'invalid_request_error' || $error_type === 'authentication_error') {
4842 - if (is_string($api_key) && $api_key !== '') {
4843 - $error_message = str_replace($api_key, '[redacted]', $error_message);
4844 - }
4845 - $error_message = sprintf(
4846 - 'Embedding failed (%s, HTTP %d): %s',
4847 - $selected_model,
4848 - $http_code,
4849 - substr($error_message, 0, 300)
4850 - );
4851 - }
2582 + //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
2583 + return $error_message;
2584 + }
4852 2585
4853 - //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4854 - return $error_message;
4855 - }
2586 + $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding", $http_code);
2587 + //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
2588 + return $error_message;
2589 + }
4856 2590
4857 - $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding for bot %s", $http_code, $bot_id);
4858 - //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
4859 - return $error_message;
4860 - }
2591 + // Parse response body
2592 + $response_body = wp_remote_retrieve_body($response);
2593 + //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
4861 2594
4862 - // Parse response body
4863 - $response_body = wp_remote_retrieve_body($response);
4864 - //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');
2595 + $response_data = json_decode($response_body, true);
4865 2596
4866 - $response_data = json_decode($response_body, true);
2597 + if (json_last_error() !== JSON_ERROR_NONE) {
2598 + $error = json_last_error_msg();
2599 + //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
2600 + //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
2601 + return "Failed to parse API response: $error";
2602 + }
4867 2603
4868 - if (json_last_error() !== JSON_ERROR_NONE) {
4869 - $error = json_last_error_msg();
4870 - //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
4871 - //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
4872 - return "Failed to parse API response: $error";
4873 - }
2604 + // Handle different response formats based on provider
2605 + if (strpos($selected_model, 'gemini-embedding') === 0) {
2606 + // Gemini API response format
2607 + if (isset($response_data['embedding']['values'])) {
2608 + $embedding_dimensions = count($response_data['embedding']['values']);
2609 + //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
4874 2610
4875 - // Handle different response formats based on provider
4876 - if (strpos($selected_model, 'gemini-embedding') === 0) {
4877 - // Gemini API response format
4878 - if (isset($response_data['embedding']['values'])) {
4879 - $embedding_dimensions = count($response_data['embedding']['values']);
4880 - //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');
2611 + // Check if embedding dimensions are as expected (should be 1536)
2612 + if ($embedding_dimensions !== 1536) {
2613 + //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
2614 + }
4881 2615
4882 - // Check if embedding dimensions are as expected (should be 1536)
4883 - if ($embedding_dimensions !== 1536) {
4884 - //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
4885 - }
2616 + return $response_data['embedding']['values'];
2617 + } else {
2618 + //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
2619 + //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4886 2620
4887 - return $response_data['embedding']['values'];
4888 - } else {
4889 - //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
4890 - //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
2621 + if (isset($response_data['error'])) {
2622 + $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
2623 + //error_log('[MXCHAT-EMBED] ' . $error_message);
2624 + return $error_message;
2625 + }
4891 2626
4892 - if (isset($response_data['error'])) {
4893 - $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
4894 - //error_log('[MXCHAT-EMBED] ' . $error_message);
4895 - return $error_message;
4896 - }
2627 + $error_message = "Invalid Gemini API response format: No embedding found";
2628 + //error_log('[MXCHAT-EMBED] ' . $error_message);
2629 + return $error_message;
2630 + }
2631 + } else {
2632 + // OpenAI/Voyage API response format
2633 + if (isset($response_data['data'][0]['embedding'])) {
2634 + $embedding_dimensions = count($response_data['data'][0]['embedding']);
2635 + //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
4897 2636
4898 - $error_message = "Invalid Gemini API response format: No embedding found";
4899 - //error_log('[MXCHAT-EMBED] ' . $error_message);
4900 - return $error_message;
4901 - }
4902 - } else {
4903 - // OpenAI/Voyage API response format
4904 - if (isset($response_data['data'][0]['embedding'])) {
4905 - $embedding_dimensions = count($response_data['data'][0]['embedding']);
4906 - //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');
2637 + // Check if embedding dimensions are as expected
2638 + if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
2639 + ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
2640 + //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
2641 + }
4907 2642
4908 - // Check if embedding dimensions are as expected
4909 - if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
4910 - ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
4911 - //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
4912 - }
2643 + return $response_data['data'][0]['embedding'];
2644 + } else {
2645 + //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
2646 + //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
4913 2647
4914 - return $response_data['data'][0]['embedding'];
4915 - } else {
4916 - //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
4917 - //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));
2648 + if (isset($response_data['error'])) {
2649 + $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
2650 + //error_log('[MXCHAT-EMBED] ' . $error_message);
2651 + return $error_message;
2652 + }
4918 2653
4919 - if (isset($response_data['error'])) {
4920 - $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
4921 - //error_log('[MXCHAT-EMBED] ' . $error_message);
4922 - return $error_message;
4923 - }
4924 -
4925 - $error_message = "Invalid API response format: No embedding found";
4926 - //error_log('[MXCHAT-EMBED] ' . $error_message);
4927 - return $error_message;
4928 - }
4929 - }
4930 -}
4931 -
4932 -/**
4933 - * Get bot-specific options for multi-bot functionality
4934 - * Falls back to default options if bot_id is 'default' or multi-bot add-on is not active
4935 - */
4936 -private function get_bot_options($bot_id = 'default') {
4937 - //error_log("MXCHAT DEBUG: get_bot_options called for bot: " . $bot_id);
4938 -
4939 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4940 - //error_log("MXCHAT DEBUG: Using default options (no multi-bot or bot is 'default')");
4941 - return array();
4942 - }
4943 -
4944 - $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
4945 -
4946 - if (!empty($bot_options)) {
4947 - //error_log("MXCHAT DEBUG: Got bot-specific options from filter");
4948 - if (isset($bot_options['similarity_threshold'])) {
4949 - //error_log(" - similarity_threshold: " . $bot_options['similarity_threshold']);
4950 - }
4951 - }
4952 -
4953 - return is_array($bot_options) ? $bot_options : array();
4954 -}
4955 -
4956 -/**
4957 - * Get bot-specific Pinecone configuration
4958 - * Used in the knowledge retrieval functions
4959 - */
4960 -// Also add debugging to your get_bot_pinecone_config function
4961 -private function get_bot_pinecone_config($bot_id = 'default') {
4962 - //error_log("MXCHAT DEBUG: get_bot_pinecone_config called for bot: " . $bot_id);
4963 -
4964 - // If default bot or multi-bot add-on not active, use default Pinecone config
4965 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
4966 - //error_log("MXCHAT DEBUG: Using default Pinecone config (no multi-bot or bot is 'default')");
4967 - $addon_options = get_option('mxchat_pinecone_addon_options', array());
4968 - $config = array(
4969 - 'use_pinecone' => (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1'),
4970 - 'api_key' => $addon_options['mxchat_pinecone_api_key'] ?? '',
4971 - 'host' => $addon_options['mxchat_pinecone_host'] ?? '',
4972 - 'namespace' => $addon_options['mxchat_pinecone_namespace'] ?? ''
4973 - );
4974 - //error_log("MXCHAT DEBUG: Default config - use_pinecone: " . ($config['use_pinecone'] ? 'true' : 'false'));
4975 - return $config;
4976 - }
4977 -
4978 - //error_log("MXCHAT DEBUG: Calling filter 'mxchat_get_bot_pinecone_config' for bot: " . $bot_id);
4979 -
4980 - // Hook for multi-bot add-on to provide bot-specific Pinecone config
4981 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
4982 -
4983 - if (!empty($bot_pinecone_config)) {
4984 - //error_log("MXCHAT DEBUG: Got bot-specific config from filter");
4985 - //error_log(" - use_pinecone: " . (isset($bot_pinecone_config['use_pinecone']) ? ($bot_pinecone_config['use_pinecone'] ? 'true' : 'false') : 'not set'));
4986 - //error_log(" - host: " . ($bot_pinecone_config['host'] ?? 'not set'));
4987 - //error_log(" - namespace: " . ($bot_pinecone_config['namespace'] ?? 'not set'));
4988 - } else {
4989 - //error_log("MXCHAT DEBUG: Filter returned empty config!");
4990 - }
4991 -
4992 - return is_array($bot_pinecone_config) ? $bot_pinecone_config : array();
4993 -}
4994 -
4995 -
2654 + $error_message = "Invalid API response format: No embedding found";
2655 + //error_log('[MXCHAT-EMBED] ' . $error_message);
2656 + return $error_message;
2657 + }
2658 + }
2659 + }
4996 2660 public function mxchat_ajax_dismiss_completed_status() {
4997 2661 try {
4998 2662 // Verify the request
4999 2663 check_ajax_referer('mxchat_status_nonce', 'nonce');
@@ -5356,569 +3020,18 @@
5356 3020
5357 3021 return $html;
5358 3022 }
5359 3023
5360 -/**
5361 - * Get all ACF fields for a specific post, excluding any fields the user has disabled
5362 - */
5363 -public function mxchat_get_acf_fields_for_post($post_id) {
5364 - if (!function_exists('get_fields')) {
5365 - return array();
5366 - }
5367 -
5368 - $fields = get_fields($post_id);
5369 - if (!$fields || !is_array($fields)) {
5370 - return array();
5371 - }
5372 -
5373 - // Get excluded fields from settings
5374 - $excluded_fields = get_option('mxchat_acf_excluded_fields', array());
5375 - if (!empty($excluded_fields) && is_array($excluded_fields)) {
5376 - foreach ($excluded_fields as $excluded_field) {
5377 - if (isset($fields[$excluded_field])) {
5378 - unset($fields[$excluded_field]);
5379 - }
5380 - }
5381 - }
5382 -
5383 - return $fields;
5384 -}
5385 -
5386 -/**
5387 - * Get all registered ACF field groups and their fields for the settings UI
5388 - */
5389 -public function mxchat_get_all_acf_fields() {
5390 - if (!function_exists('acf_get_field_groups') || !function_exists('acf_get_fields')) {
5391 - return array();
5392 - }
5393 -
5394 - $all_fields = array();
5395 - $field_groups = acf_get_field_groups();
5396 -
5397 - if (!empty($field_groups)) {
5398 - foreach ($field_groups as $group) {
5399 - $group_fields = acf_get_fields($group['key']);
5400 - if (!empty($group_fields)) {
5401 - $all_fields[$group['title']] = array();
5402 - foreach ($group_fields as $field) {
5403 - $all_fields[$group['title']][] = array(
5404 - 'name' => $field['name'],
5405 - 'label' => $field['label'],
5406 - 'type' => $field['type']
5407 - );
5408 - }
5409 - }
5410 - }
5411 - }
5412 -
5413 - return $all_fields;
5414 -}
5415 -
5416 -/**
5417 - * Get whitelisted custom post meta for a given post
5418 - * This allows non-ACF meta fields (like OptionTree, theme meta boxes) to be included in embeddings
5419 - */
5420 -public function mxchat_get_whitelisted_post_meta($post_id) {
5421 - $whitelist = get_option('mxchat_custom_meta_whitelist', '');
5422 -
5423 - if (empty($whitelist)) {
5424 - return array();
5425 - }
5426 -
5427 - // Parse the whitelist - one meta key per line
5428 - $meta_keys = array_filter(array_map('trim', explode("\n", $whitelist)));
5429 -
5430 - if (empty($meta_keys)) {
5431 - return array();
5432 - }
5433 -
5434 - $result = array();
5435 -
5436 - foreach ($meta_keys as $key) {
5437 - // Skip empty keys
5438 - if (empty($key)) {
5439 - continue;
5440 - }
5441 -
5442 - $value = get_post_meta($post_id, $key, true);
5443 -
5444 - // Only include non-empty string values
5445 - if (!empty($value) && is_string($value)) {
5446 - $result[$key] = $value;
5447 - } elseif (!empty($value) && is_array($value)) {
5448 - // Handle array values by joining them
5449 - $flat_value = $this->mxchat_flatten_meta_array($value);
5450 - if (!empty($flat_value)) {
5451 - $result[$key] = $flat_value;
5452 - }
5453 - }
5454 - }
5455 -
5456 - return $result;
5457 -}
5458 -
5459 -/**
5460 - * Flatten array meta values into a readable string
5461 - */
5462 -private function mxchat_flatten_meta_array($array, $depth = 0) {
5463 - if ($depth > 3) {
5464 - return ''; // Prevent infinite recursion
5465 - }
5466 -
5467 - $parts = array();
5468 -
5469 - foreach ($array as $key => $value) {
5470 - if (is_string($value) && !empty($value)) {
5471 - $parts[] = $value;
5472 - } elseif (is_array($value)) {
5473 - $nested = $this->mxchat_flatten_meta_array($value, $depth + 1);
5474 - if (!empty($nested)) {
5475 - $parts[] = $nested;
5476 - }
5477 - }
5478 - }
5479 -
5480 - return implode(', ', $parts);
5481 -}
5482 -
5483 -/**
5484 - * Format ACF field values for content extraction
5485 - */
5486 -public function mxchat_format_acf_field_value($value, $field_name = '', $post_id = 0) {
5487 - if (empty($value)) {
5488 - return '';
5489 - }
5490 -
5491 - // Handle WP_Post objects first (THIS IS THE KEY FIX)
5492 - if ($value instanceof WP_Post) {
5493 - return $value->post_title ?: '';
5494 - }
5495 -
5496 - // Handle other WP objects
5497 - if (is_object($value)) {
5498 - if (isset($value->post_title)) {
5499 - return $value->post_title;
5500 - } elseif (isset($value->display_name)) {
5501 - return $value->display_name;
5502 - } elseif (isset($value->name)) {
5503 - return $value->name;
5504 - } elseif (method_exists($value, '__toString')) {
5505 - try {
5506 - return (string) $value;
5507 - } catch (Exception $e) {
5508 - return '';
5509 - }
5510 - }
5511 - // For any other objects, return empty string
5512 - return '';
5513 - }
5514 -
5515 - // Handle different ACF field types
5516 - if (is_array($value)) {
5517 - // Check if it's an image/file field
5518 - if (isset($value['url'])) {
5519 - // Image field - return alt text, title, or caption
5520 - if (!empty($value['alt'])) {
5521 - return $value['alt'];
5522 - } elseif (!empty($value['title'])) {
5523 - return $value['title'];
5524 - } elseif (!empty($value['caption'])) {
5525 - return $value['caption'];
5526 - } else {
5527 - return ''; // Don't include just the URL
5528 - }
5529 - }
5530 -
5531 - // Check if it's a post object or relationship field
5532 - if (isset($value['post_title'])) {
5533 - return $value['post_title'];
5534 - }
5535 -
5536 - // Check if it's a user field
5537 - if (isset($value['display_name'])) {
5538 - return $value['display_name'];
5539 - }
5540 -
5541 - // Check if it's a taxonomy term
5542 - if (isset($value['name']) && isset($value['taxonomy'])) {
5543 - return $value['name'];
5544 - }
5545 -
5546 - // Check if it's a select field with label
5547 - if (isset($value['label'])) {
5548 - return $value['label'];
5549 - }
5550 -
5551 - // Check for repeater field or flexible content
5552 - if (is_numeric(key($value))) {
5553 - $sub_values = array();
5554 - foreach ($value as $sub_item) {
5555 - if (is_array($sub_item)) {
5556 - // For repeater/flexible content, extract text values
5557 - $sub_text = $this->mxchat_extract_text_from_acf_array($sub_item);
5558 - if (!empty($sub_text)) {
5559 - $sub_values[] = $sub_text;
5560 - }
5561 - } elseif ($sub_item instanceof WP_Post) {
5562 - // Handle WP_Post objects in arrays
5563 - $sub_values[] = $sub_item->post_title ?: '';
5564 - } else {
5565 - $sub_values[] = (string) $sub_item;
5566 - }
5567 - }
5568 - return implode(', ', array_filter($sub_values));
5569 - }
5570 -
5571 - // For other arrays, try to extract meaningful text
5572 - $text_values = array();
5573 - foreach ($value as $key => $val) {
5574 - if (is_string($val) && !empty(trim($val))) {
5575 - $text_values[] = trim($val);
5576 - } elseif ($val instanceof WP_Post) {
5577 - // Handle WP_Post objects in associative arrays
5578 - $text_values[] = $val->post_title ?: '';
5579 - } elseif (is_array($val) && isset($val['post_title'])) {
5580 - $text_values[] = $val['post_title'];
5581 - } elseif (is_array($val) && isset($val['name'])) {
5582 - $text_values[] = $val['name'];
5583 - }
5584 - }
5585 -
5586 - return implode(', ', array_filter($text_values));
5587 - }
5588 -
5589 - // Handle boolean values
5590 - if (is_bool($value)) {
5591 - return $value ? 'Yes' : 'No';
5592 - }
5593 -
5594 - // Handle numeric values
5595 - if (is_numeric($value)) {
5596 - return (string) $value;
5597 - }
5598 -
5599 - // Handle string values
5600 - if (is_string($value)) {
5601 - return trim($value);
5602 - }
5603 -
5604 - // For anything else that we can't handle, return empty string
5605 - // This prevents the "Object could not be converted to string" error
5606 - return '';
5607 -}
5608 -
5609 -/**
5610 - * Extract text from complex ACF array structures
5611 - */
5612 -private function mxchat_extract_text_from_acf_array($array) {
5613 - if (!is_array($array)) {
5614 - return '';
5615 - }
5616 -
5617 - $text_parts = array();
5618 -
5619 - foreach ($array as $key => $value) {
5620 - if (is_string($value) && !empty(trim($value))) {
5621 - // Skip keys that are likely to be IDs or technical values
5622 - if (!is_numeric($value) || strlen($value) > 10) {
5623 - $text_parts[] = trim($value);
5624 - }
5625 - } elseif ($value instanceof WP_Post) {
5626 - // Handle WP_Post objects
5627 - $text_parts[] = $value->post_title ?: '';
5628 - } elseif (is_array($value)) {
5629 - if (isset($value['post_title'])) {
5630 - $text_parts[] = $value['post_title'];
5631 - } elseif (isset($value['name'])) {
5632 - $text_parts[] = $value['name'];
5633 - } elseif (isset($value['label'])) {
5634 - $text_parts[] = $value['label'];
5635 - }
5636 - } elseif (is_object($value)) {
5637 - // Handle other objects safely
5638 - if (isset($value->post_title)) {
5639 - $text_parts[] = $value->post_title;
5640 - } elseif (isset($value->name)) {
5641 - $text_parts[] = $value->name;
5642 - } elseif (isset($value->display_name)) {
5643 - $text_parts[] = $value->display_name;
5644 - }
5645 - }
5646 - }
5647 -
5648 - return implode(', ', array_filter($text_parts));
5649 -}
5650 -
5651 -/**
5652 - * Walk an ACF field value tree and collect attachment IDs for any value that
5653 - * resolves to a PDF in the WordPress media library. Handles the three shapes
5654 - * ACF returns for File/Image/URL fields (array with ID+url, integer attachment ID,
5655 - * plain URL string), and recurses through repeater/group/flexible content.
5656 - *
5657 - * @param mixed $value The ACF field value (any depth)
5658 - * @param array $out Accumulator (passed by reference) for attachment IDs
5659 - * @param int $depth Recursion guard
5660 - */
5661 -private function mxchat_collect_pdf_attachment_ids_from_acf_value($value, &$out, $depth = 0) {
5662 - if ($depth > 6) {
5663 - return; // prevent runaway recursion on circular/very-deep structures
5664 - }
5665 -
5666 - if (empty($value)) {
5667 - return;
5668 - }
5669 -
5670 - // Array shapes: ACF File/Image return value=array; repeaters/groups are arrays of arrays
5671 - if (is_array($value)) {
5672 - // Direct File/Image-style array (has 'url' and usually 'ID' + 'mime_type')
5673 - $looks_like_attachment = isset($value['url']) || isset($value['ID']) || isset($value['id']);
5674 - if ($looks_like_attachment) {
5675 - $att_id = 0;
5676 - if (!empty($value['ID']) && is_numeric($value['ID'])) {
5677 - $att_id = (int) $value['ID'];
5678 - } elseif (!empty($value['id']) && is_numeric($value['id'])) {
5679 - $att_id = (int) $value['id'];
5680 - } elseif (!empty($value['url']) && is_string($value['url'])) {
5681 - $att_id = (int) attachment_url_to_postid($value['url']);
5682 - }
5683 -
5684 - $is_pdf = false;
5685 - if (!empty($value['mime_type']) && $value['mime_type'] === 'application/pdf') {
5686 - $is_pdf = true;
5687 - } elseif (!empty($value['subtype']) && strtolower((string) $value['subtype']) === 'pdf') {
5688 - $is_pdf = true;
5689 - } elseif (!empty($value['url']) && is_string($value['url']) && $this->mxchat_url_looks_like_pdf($value['url'])) {
5690 - $is_pdf = true;
5691 - } elseif ($att_id && get_post_mime_type($att_id) === 'application/pdf') {
5692 - $is_pdf = true;
5693 - }
5694 -
5695 - if ($is_pdf && $att_id && get_post_mime_type($att_id) === 'application/pdf') {
5696 - $out[] = $att_id;
5697 - }
5698 - // An array node that represents one attachment doesn't contain other
5699 - // attachments inside it — done with this branch.
5700 - return;
5701 - }
5702 -
5703 - // Recurse: repeater rows, flexible-content layouts, groups, etc.
5704 - foreach ($value as $sub) {
5705 - $this->mxchat_collect_pdf_attachment_ids_from_acf_value($sub, $out, $depth + 1);
5706 - }
5707 - return;
5708 - }
5709 -
5710 - // Plain numeric attachment ID (ACF File field set to "Return: ID")
5711 - if (is_numeric($value)) {
5712 - $att_id = (int) $value;
5713 - if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5714 - $out[] = $att_id;
5715 - }
5716 - return;
5717 - }
5718 -
5719 - // Plain string — URL pointing at a PDF (ACF File field set to "Return: URL", or a custom URL/text field)
5720 - if (is_string($value)) {
5721 - $trimmed = trim($value);
5722 - if ($trimmed !== '' && $this->mxchat_url_looks_like_pdf($trimmed)) {
5723 - $att_id = (int) attachment_url_to_postid($trimmed);
5724 - if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
5725 - $out[] = $att_id;
5726 - }
5727 - }
5728 - return;
5729 - }
5730 -}
5731 -
5732 -/**
5733 - * Heuristic: does this URL/string look like a PDF reference?
5734 - * Tolerates query strings and fragments (#page=2).
5735 - */
5736 -private function mxchat_url_looks_like_pdf($url) {
5737 - if (!is_string($url) || $url === '') {
5738 - return false;
5739 - }
5740 - // Strip query + fragment before checking extension
5741 - $path = preg_replace('/[?#].*$/', '', $url);
5742 - return (bool) preg_match('/\.pdf$/i', $path);
5743 -}
5744 -
5745 -/**
5746 - * Extract text from a PDF attachment by ID using the bundled Smalot parser.
5747 - * Reads the file directly from disk via get_attached_file (no HTTP fetch).
5748 - * Result is cached on the attachment as post_meta keyed by file mtime so we
5749 - * only parse the same PDF once unless the file changes on disk.
5750 - *
5751 - * @param int $attachment_id
5752 - * @return string Extracted plain text, or '' on failure.
5753 - */
5754 -private function mxchat_extract_pdf_text_by_attachment_id($attachment_id) {
5755 - $attachment_id = (int) $attachment_id;
5756 - if ($attachment_id <= 0) {
5757 - return '';
5758 - }
5759 - if (get_post_mime_type($attachment_id) !== 'application/pdf') {
5760 - return '';
5761 - }
5762 -
5763 - $pdf_path = get_attached_file($attachment_id);
5764 - if (empty($pdf_path) || !file_exists($pdf_path) || !is_readable($pdf_path)) {
5765 - return '';
5766 - }
5767 -
5768 - // Raw-file size cap. Parsing very large PDFs can OOM the request; skip with a log entry
5769 - // and let the rest of the ACF content land in the KB. Filterable for users who need it bigger.
5770 - $default_max_bytes = 25 * 1024 * 1024;
5771 - $max_bytes = (int) apply_filters('mxchat_acf_pdf_max_bytes', $default_max_bytes, $attachment_id, $pdf_path);
5772 - if ($max_bytes > 0) {
5773 - $file_size = @filesize($pdf_path);
5774 - if ($file_size !== false && $file_size > $max_bytes) {
5775 - error_log(sprintf(
5776 - '[mxchat] ACF PDF skipped (over size cap): attachment %d "%s" %d bytes > cap %d',
5777 - $attachment_id,
5778 - basename($pdf_path),
5779 - $file_size,
5780 - $max_bytes
5781 - ));
5782 - return '';
5783 - }
5784 - }
5785 -
5786 - $mtime = @filemtime($pdf_path);
5787 - $cache_meta_key = '_mxchat_acf_pdf_text_v1';
5788 - $cached = get_post_meta($attachment_id, $cache_meta_key, true);
5789 - if (is_array($cached) && isset($cached['mtime'], $cached['text']) && (int) $cached['mtime'] === (int) $mtime) {
5790 - return (string) $cached['text'];
5791 - }
5792 -
5793 - $text = '';
5794 - try {
5795 - if (function_exists('mxchat_load_pdf_parser')) {
5796 - mxchat_load_pdf_parser();
5797 - }
5798 - if (!class_exists('\\Smalot\\PdfParser\\Parser')) {
5799 - return '';
5800 - }
5801 - $parser = new \Smalot\PdfParser\Parser();
5802 - $pdf = $parser->parseFile($pdf_path);
5803 - $pages = $pdf->getPages();
5804 - $page_texts = array();
5805 - foreach ($pages as $page) {
5806 - $page_text = '';
5807 - try {
5808 - $page_text = $page->getText();
5809 - } catch (\Exception $e) {
5810 - $page_text = '';
5811 - }
5812 - if (!empty($page_text)) {
5813 - $page_texts[] = $page_text;
5814 - }
5815 - }
5816 - $text = trim(implode("\n\n", $page_texts));
5817 - } catch (\Exception $e) {
5818 - error_log('[mxchat] ACF PDF extraction failed for attachment ' . $attachment_id . ': ' . $e->getMessage());
5819 - return '';
5820 - } catch (\Throwable $e) {
5821 - error_log('[mxchat] ACF PDF extraction error for attachment ' . $attachment_id . ': ' . $e->getMessage());
5822 - return '';
5823 - }
5824 -
5825 - // Cap per-PDF text to avoid blowing up the embedding payload on enormous PDFs.
5826 - // The chunker downstream will still split this into multiple vectors.
5827 - $max_len = (int) apply_filters('mxchat_acf_pdf_text_max_length', 50000);
5828 - if ($max_len > 0 && strlen($text) > $max_len) {
5829 - $text = substr($text, 0, $max_len);
5830 - }
5831 -
5832 - update_post_meta($attachment_id, $cache_meta_key, array(
5833 - 'mtime' => (int) $mtime,
5834 - 'text' => $text,
5835 - ));
5836 -
5837 - return $text;
5838 -}
5839 -
5840 -/**
5841 - * Handle ACF save - fires after ACF fields are saved
5842 - * This ensures ACF field data is available when syncing to knowledge base
5843 - */
5844 -public function mxchat_handle_acf_save($post_id) {
5845 - // Skip if not a valid post
5846 - if (!$post_id || $post_id === 'options') {
5847 - return;
5848 - }
5849 -
5850 - // Skip autosaves and revisions
3024 +public function mxchat_handle_post_update($post_id, $post, $update) {
3025 + // Basic validation checks
5851 3026 if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5852 3027 return;
5853 3028 }
5854 -
5855 - $post = get_post($post_id);
5856 - if (!$post) {
5857 - return;
5858 - }
5859 -
5860 - $post_type = $post->post_type;
5861 -
5862 - // Check if sync is enabled for this post type
5863 - $should_sync = false;
5864 -
5865 - if ($post_type === 'post' && get_option('mxchat_auto_sync_posts') === '1') {
5866 - $should_sync = true;
5867 - } else if ($post_type === 'page' && get_option('mxchat_auto_sync_pages') === '1') {
5868 - $should_sync = true;
5869 - } else if ($post_type === 'product' && class_exists('WooCommerce')) {
5870 - // WooCommerce products - check if WooCommerce integration is enabled
5871 - $options = get_option('mxchat_options', array());
5872 - if (isset($options['enable_woocommerce_integration']) &&
5873 - ($options['enable_woocommerce_integration'] === '1' || $options['enable_woocommerce_integration'] === 'on')) {
5874 - $should_sync = true;
5875 - }
5876 - } else {
5877 - // Check custom post types
5878 - $option_name = 'mxchat_auto_sync_' . $post_type;
5879 - if (get_option($option_name) === '1') {
5880 - $should_sync = true;
5881 - }
5882 - }
5883 -
5884 - if (!$should_sync) {
5885 - return;
5886 - }
5887 -
5888 - // Only process published posts
3029 +
3030 + // Only process published content
5889 3031 if ($post->post_status !== 'publish') {
5890 3032 return;
5891 3033 }
5892 -
5893 - // Check if this post has any ACF fields - if not, no need to re-sync
5894 - $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
5895 - if (empty($acf_fields)) {
5896 - return;
5897 - }
5898 -
5899 - // Use a transient to prevent duplicate processing (post_updated may have already run)
5900 - $transient_key = 'mxchat_acf_synced_' . $post_id;
5901 - if (get_transient($transient_key)) {
5902 - return;
5903 - }
5904 - set_transient($transient_key, true, 60); // Prevent re-processing for 60 seconds
5905 -
5906 - // Re-run the sync with ACF data now available
5907 - // We pass $update=true since this is effectively an update with ACF data
5908 - $this->mxchat_handle_post_update($post_id, $post, true);
5909 -}
5910 -
5911 -public function mxchat_handle_post_update($post_id, $post, $update) {
5912 - // The in-flight-update marker has done its job the moment post_updated runs; drop it
5913 - // before any early return so it can never outlive its own save (a failed $wpdb->update
5914 - // inside wp_insert_post returns after pre_post_update but before the transition).
5915 - unset($this->pending_post_update[$post_id]);
5916 -
5917 - // Basic validation checks
5918 - if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
5919 - return;
5920 - }
5921 3034
5922 3035 $post_type = $post->post_type;
5923 3036
5924 3037 // Check if sync is enabled for this post type
@@ -5940,518 +3053,80 @@
5940 3053 if (!$should_sync) {
5941 3054 return;
5942 3055 }
5943 3056
5944 - // Check if we have stored the previous status and URL in our transients
5945 - $previous_status_key = 'mxchat_prev_status_' . $post_id;
5946 - $previous_status = get_transient($previous_status_key);
3057 + // Get content with proper formatting (matching ajax_mxchat_process_selected_content)
3058 + $title = get_the_title($post_id);
3059 + $content = get_post_field('post_content', $post_id);
5947 3060
5948 - $previous_url_key = 'mxchat_prev_url_' . $post_id;
5949 - $previous_url = get_transient($previous_url_key);
3061 + // Apply WordPress content filters to get properly formatted content
3062 + $content = apply_filters('the_content', $content);
5950 3063
5951 - // If the post was previously published but is now not published, remove from knowledge base
5952 - if ($previous_status === 'publish' && $post->post_status !== 'publish') {
5953 - // Use the stored URL from when it was published, or fall back to current permalink
5954 - $source_url = $previous_url ?: get_permalink($post_id);
5955 -
5956 - // mxchat_handle_status_transition already deleted for this post earlier in this
5957 - // request (it fires first inside wp_insert_post); skip the redundant round-trip.
5958 - if ($source_url && empty($this->transition_deleted_posts[$post_id])) {
5959 - // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
5960 - MxChat_Utils::delete_chunks_for_url($source_url, 'default');
5961 - }
5962 -
5963 - // Clean up the transients and exit early
5964 - delete_transient($previous_status_key);
5965 - delete_transient($previous_url_key);
5966 - return;
5967 - }
5968 -
5969 - // Slug/permalink rename while still published: delete the old vectors before upserting new ones.
5970 - // Without this, md5(old_url) vectors (base + chunks) would be orphaned under the stale URL.
5971 - if ($post->post_status === 'publish' && !empty($previous_url)) {
5972 - $current_url = get_permalink($post_id);
5973 - if ($current_url && $current_url !== $previous_url) {
5974 - MxChat_Utils::delete_chunks_for_url($previous_url, 'default');
5975 - }
5976 - }
5977 -
5978 - // Store the current status for next time (if this is an update)
5979 - if ($update) {
5980 - set_transient($previous_status_key, $post->post_status, DAY_IN_SECONDS);
5981 -
5982 - // If the post is currently published, also store its URL
5983 - if ($post->post_status === 'publish') {
5984 - $current_url = get_permalink($post_id);
5985 - set_transient($previous_url_key, $current_url, DAY_IN_SECONDS);
5986 - }
5987 - }
3064 + // Strip tags but preserve structure
3065 + $content = wp_strip_all_tags($content);
5988 3066
5989 - // Only process currently published content for adding/updating.
5990 - // transition_indexed_posts: mxchat_handle_status_transition's arrival edge may have
5991 - // already indexed this post earlier in this request (editor publishes fire
5992 - // transition_post_status first, then post_updated) — skip the duplicate embed.
5993 - // Consume-once: the flag is cleared when honoured, so a LATER save of the same
5994 - // post in one long-running process (WP-CLI scripts, importers) re-indexes normally.
5995 - if ($post->post_status === 'publish') {
5996 - if (!empty($this->transition_indexed_posts[$post_id])) {
5997 - unset($this->transition_indexed_posts[$post_id]);
5998 - } else {
5999 - $this->mxchat_index_published_post($post_id, $post);
3067 + // Combine title and content
3068 + $final_content = $title . "\n\n" . $content;
3069 +
3070 + // For custom post types like job_listing, include additional fields
3071 + if ($post_type === 'job_listing') {
3072 + // Add job-specific meta if available
3073 + $job_location = get_post_meta($post_id, '_job_location', true);
3074 + if (!empty($job_location)) {
3075 + $final_content .= "\n\nLocation: " . $job_location;
6000 3076 }
6001 - }
6002 -
6003 - // Clean up the stored previous status if not used above
6004 - if ($previous_status !== 'publish' || $post->post_status === 'publish') {
6005 - delete_transient($previous_status_key);
6006 - delete_transient($previous_url_key);
6007 - }
6008 -}
6009 -
6010 -/**
6011 - * Index a published post into the knowledge base: preprocessing filter, content
6012 - * assembly (title/excerpt/body), WooCommerce product enrichment, job_listing meta,
6013 - * ACF fields (+ optional PDF extraction), whitelisted custom meta, embedding and
6014 - * upsert, then tag-based role restriction.
6015 - *
6016 - * Shared by the post_updated auto-sync path (mxchat_handle_post_update) and the
6017 - * transition_post_status arrival edge (mxchat_handle_status_transition), so
6018 - * scheduled publishes (wp_publish_post) and direct status=publish inserts index
6019 - * identically to editor saves (plan 3055e1). Pure extraction of the former
6020 - * publish branch — body indentation retained to keep the diff reviewable.
6021 - */
6022 -private function mxchat_index_published_post($post_id, $post) {
6023 - $post_type = $post->post_type;
6024 -
6025 - // Get the source URL
6026 - $source_url = get_permalink($post_id);
6027 -
6028 - /**
6029 - * Allow developers to modify post data before processing into the knowledge base.
6030 - * Same filter and signature as the manual bulk-import path
6031 - * (ajax_mxchat_process_selected_content), so a callback registered once covers
6032 - * every indexing route. Purely additive — zero behaviour change when unhooked.
6033 - * Auto-sync runs under the 'default' bot context, matching the rest of this
6034 - * function.
6035 - *
6036 - * @param WP_Post $post The post about to be indexed.
6037 - * @param string $bot_id Bot context ('default' on auto-sync).
6038 - */
6039 - $post = apply_filters('mxchat_before_process_post', $post, 'default');
6040 - if (!($post instanceof WP_Post)) {
6041 - $post = get_post($post_id); // defend against a bad callback return
6042 - }
6043 -
6044 - // Get content with proper formatting (matching ajax_mxchat_process_selected_content),
6045 - // reading from the FILTERED post object — not re-fetched by ID, which would discard it
6046 - // Raw post_title, NOT get_the_title(): the_title applies wptexturize +
6047 - // convert_chars (curly quotes and em-dashes become HTML entities in the
6048 - // embedded string) and prepends the "Protected:" / "Private:" display
6049 - // chrome. The knowledge base stores facts, not display strings — and the
6050 - // bulk-import path has always read the raw title, so this is also what
6051 - // makes the two paths agree.
6052 - $title = $post->post_title;
6053 - $content = get_post_field('post_content', $post);
6054 - $excerpt = get_post_field('post_excerpt', $post);
6055 -
6056 - // Remove shortcode tags but preserve content inside them
6057 - $content = $this->strip_shortcode_tags_preserve_content($content);
6058 - $excerpt = $this->strip_shortcode_tags_preserve_content($excerpt);
6059 -
6060 - // Strip tags but preserve structure (don't use 'the_content' filter as it may re-add shortcodes)
6061 - $content = wp_strip_all_tags($content);
6062 -
6063 - // Combine title, short description (if exists), and content
6064 - $final_content = $title . "\n\n";
6065 -
6066 - // Add short description if it exists (WooCommerce products use post_excerpt for short description)
6067 - // trim() only in the TEST — see the matching note on the bulk-import path.
6068 - if (trim($excerpt) !== '') {
6069 - $final_content .= "Short Description: " . wp_strip_all_tags($excerpt) . "\n\n";
6070 - }
6071 -
6072 - $final_content .= $content;
6073 -
6074 - // For WooCommerce products, include pricing and product details
6075 - if ($post_type === 'product' && class_exists('WooCommerce')) {
6076 - $product = wc_get_product($post_id);
6077 -
6078 - if ($product) {
6079 - // Get pricing information
6080 - $regular_price = $product->get_regular_price();
6081 - $sale_price = $product->get_sale_price();
6082 - $price = $product->get_price();
6083 - $sku = $product->get_sku();
6084 -
6085 - // Get currency symbol
6086 - $currency_symbol = get_woocommerce_currency_symbol();
6087 -
6088 - // Add pricing information
6089 - $final_content .= "\n";
6090 - if (!empty($regular_price)) {
6091 - $final_content .= "Price: " . $currency_symbol . $regular_price . "\n";
6092 - } elseif (!empty($price)) {
6093 - $final_content .= "Price: " . $currency_symbol . $price . "\n";
6094 - }
6095 -
6096 - if (!empty($sale_price) && $sale_price !== $regular_price) {
6097 - $final_content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
6098 - }
6099 -
6100 - // Handle variable products - show price range
6101 - if ($product->is_type('variable')) {
6102 - $min_price = $product->get_variation_price('min');
6103 - $max_price = $product->get_variation_price('max');
6104 - if ($min_price !== $max_price) {
6105 - $final_content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
6106 - }
6107 - }
6108 -
6109 - if (!empty($sku)) {
6110 - $final_content .= "SKU: " . $sku . "\n";
6111 - }
6112 -
6113 - // Get product categories
6114 - $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names'));
6115 - if (!empty($categories) && !is_wp_error($categories)) {
6116 - $final_content .= "Categories: " . implode(', ', $categories) . "\n";
6117 - }
3077 +
3078 + // Get job type terms
3079 + $job_types = get_the_terms($post_id, 'job_listing_type');
3080 + if (!empty($job_types) && !is_wp_error($job_types)) {
3081 + $types = array();
3082 + foreach ($job_types as $type) {
3083 + $types[] = $type->name;
6118 3084 }
3085 + $final_content .= "\n\nJob Type: " . implode(', ', $types);
6119 3086 }
6120 -
6121 - // For custom post types like job_listing, include additional fields
6122 - if ($post_type === 'job_listing') {
6123 - // Add job-specific meta if available
6124 - $job_location = get_post_meta($post_id, '_job_location', true);
6125 - if (!empty($job_location)) {
6126 - $final_content .= "\n\nLocation: " . $job_location;
6127 - }
6128 -
6129 - // Get job type terms
6130 - $job_types = get_the_terms($post_id, 'job_listing_type');
6131 - if (!empty($job_types) && !is_wp_error($job_types)) {
6132 - $types = array();
6133 - foreach ($job_types as $type) {
6134 - $types[] = $type->name;
6135 - }
6136 - $final_content .= "\n\nJob Type: " . implode(', ', $types);
6137 - }
6138 -
6139 - // Get company name if available
6140 - $company_name = get_post_meta($post_id, '_company_name', true);
6141 - if (!empty($company_name)) {
6142 - $final_content .= "\n\nCompany: " . $company_name;
6143 - }
6144 - }
6145 -
6146 - // ADD ACF FIELDS SUPPORT (matches ajax_mxchat_process_selected_content behavior)
6147 - $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
6148 - if (!empty($acf_fields)) {
6149 - $acf_content_parts = array();
6150 - $pdf_attachment_ids = array();
6151 -
6152 - foreach ($acf_fields as $field_name => $field_value) {
6153 - $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);
6154 - if (!empty($formatted_value)) {
6155 - // Convert field name to readable label
6156 - $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name));
6157 - $acf_content_parts[] = $field_label . ": " . $formatted_value;
6158 - }
6159 -
6160 - $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
6161 - }
6162 -
6163 - if (!empty($acf_content_parts)) {
6164 - $final_content .= "\n\n" . implode("\n", $acf_content_parts);
6165 - }
6166 -
6167 - // Gate the auto-sync PDF-extraction loop behind an opt-in option.
6168 - // Mirrors the per-batch checkbox the manual content selector has; the
6169 - // 25 MB size cap lives in the shared extractor so it applies in both
6170 - // paths regardless. Default OFF — re-parsing every ACF PDF on every
6171 - // editor save is expensive and most sites don't want it.
6172 - $autosync_extract_acf_pdfs = get_option('mxchat_auto_sync_acf_pdfs', '0') === '1';
6173 - if ($autosync_extract_acf_pdfs && !empty($pdf_attachment_ids)) {
6174 - $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
6175 - $pdf_sections = array();
6176 - foreach ($pdf_attachment_ids as $att_id) {
6177 - $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
6178 - if (!empty($pdf_text)) {
6179 - $pdf_title = get_the_title($att_id);
6180 - $pdf_url = wp_get_attachment_url($att_id);
6181 - $header = 'PDF Attachment';
6182 - if (!empty($pdf_title)) {
6183 - $header .= ': ' . $pdf_title;
6184 - }
6185 - if (!empty($pdf_url)) {
6186 - $header .= ' (' . $pdf_url . ')';
6187 - }
6188 - $pdf_sections[] = $header . "\n" . $pdf_text;
6189 - }
6190 - }
6191 - if (!empty($pdf_sections)) {
6192 - $final_content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
6193 - }
6194 - }
6195 - }
6196 -
6197 - // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
6198 - $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
6199 - if (!empty($custom_meta)) {
6200 - $meta_content_parts = array();
6201 -
6202 - foreach ($custom_meta as $meta_key => $meta_value) {
6203 - // Convert meta key to readable label
6204 - $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
6205 - $meta_content_parts[] = $meta_label . ": " . $meta_value;
6206 - }
6207 -
6208 - if (!empty($meta_content_parts)) {
6209 - $final_content .= "\n\n" . implode("\n", $meta_content_parts);
6210 - }
6211 - }
6212 -
6213 - // Embedding decision — custom-provider-aware. Gating on a cloud API key
6214 - // here silently killed auto-sync on keyless custom-embeddings sites,
6215 - // because generate_embedding() routes custom FIRST and never needs the
6216 - // key (plan cbd5fd). Silent-return shape preserved.
6217 - $preflight = MxChat_Utils::embedding_preflight(get_option('mxchat_options'));
6218 - if (!$preflight['ok']) {
6219 - return;
6220 - }
6221 - $api_key = $preflight['api_key'];
6222 -
6223 - // Use the centralized utility function for storage
6224 - $result = MxChat_Utils::submit_content_to_db(
6225 - $final_content,
6226 - $source_url,
6227 - $api_key,
6228 - md5($source_url) // Vector ID for Pinecone
6229 - );
6230 3087
6231 - // After successful storage, apply role restriction based on tags
6232 - if (!is_wp_error($result)) {
6233 - $this->apply_role_restriction_to_post($post_id, $source_url);
3088 + // Get company name if available
3089 + $company_name = get_post_meta($post_id, '_company_name', true);
3090 + if (!empty($company_name)) {
3091 + $final_content .= "\n\nCompany: " . $company_name;
6234 3092 }
6235 -}
6236 -
6237 -/**
6238 - * Store the post status and URL before update to detect status transitions
6239 - * This runs before the post is actually updated in the database
6240 - */
6241 -public function mxchat_store_pre_update_status($post_id, $data) {
6242 - // Core is inside wp_insert_post's update branch, so a post_updated WILL fire later
6243 - // this request and can consume the arrival-edge guard (plan a664f3).
6244 - $this->pending_post_update[$post_id] = true;
6245 -
6246 - // Get the current post from database (before update)
6247 - $current_post = get_post($post_id);
3093 + }
6248 3094
6249 - if ($current_post) {
6250 - // Store the current status temporarily
6251 - $status_key = 'mxchat_prev_status_' . $post_id;
6252 - set_transient($status_key, $current_post->post_status, HOUR_IN_SECONDS);
6253 -
6254 - // If the post is currently published, also store its URL
6255 - if ($current_post->post_status === 'publish') {
6256 - $url_key = 'mxchat_prev_url_' . $post_id;
6257 - $current_url = get_permalink($post_id);
6258 - set_transient($url_key, $current_url, HOUR_IN_SECONDS);
6259 - }
3095 + // Get the source URL
3096 + $source_url = get_permalink($post_id);
3097 +
3098 + // Get API key with proper model detection
3099 + $options = get_option('mxchat_options');
3100 + $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
3101 +
3102 + if (strpos($selected_model, 'voyage') === 0) {
3103 + $api_key = $options['voyage_api_key'] ?? '';
3104 + } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
3105 + $api_key = $options['gemini_api_key'] ?? '';
3106 + } else {
3107 + $api_key = $options['api_key'] ?? '';
6260 3108 }
6261 -}
6262 -
6263 -/**
6264 - * Whether auto-sync is enabled for a post type (mirrors the checks used by the
6265 - * update/delete handlers; kept as one helper so new call sites cannot drift).
6266 - */
6267 -private function mxchat_is_auto_sync_enabled($post_type) {
6268 - if ($post_type === 'post') {
6269 - return get_option('mxchat_auto_sync_posts') === '1';
6270 - }
6271 - if ($post_type === 'page') {
6272 - return get_option('mxchat_auto_sync_pages') === '1';
6273 - }
6274 - return get_option('mxchat_auto_sync_' . $post_type) === '1';
6275 -}
6276 -
6277 -/**
6278 - * Remove a post's vectors the moment it leaves 'publish', using the authoritative
6279 - * old status core passes to transition_post_status — no transient involved (plan 816fb1).
6280 - *
6281 - * Covers status changes that never route through wp_update_post (scheduled-expiry
6282 - * plugins and others that flip post_status directly and call wp_transition_post_status),
6283 - * where neither pre_post_update nor post_updated fires and the old detection missed.
6284 - */
6285 -public function mxchat_handle_status_transition($new_status, $old_status, $post) {
6286 - if (!($post instanceof WP_Post) || wp_is_post_revision($post->ID)) {
3109 +
3110 + if (empty($api_key)) {
3111 + //error_log('MxChat Auto-sync: No API key configured for embedding model');
6287 3112 return;
6288 3113 }
6289 -
6290 - // Arrival edge (plan 3055e1): a post BECOMING published is indexed here, because
6291 - // wp_publish_post() — the path scheduled posts take via check_and_publish_future_post —
6292 - // and direct wp_insert_post(status=publish) creates never fire post_updated, so the
6293 - // auto-sync ADD path alone misses them. Editor publishes also pass through here;
6294 - // the transition_indexed_posts guard keeps mxchat_handle_post_update from embedding
6295 - // a second time in the same request.
6296 - if ($new_status === 'publish' && $old_status !== 'publish') {
6297 - if ($this->mxchat_is_auto_sync_enabled($post->post_type)) {
6298 - $this->mxchat_index_published_post($post->ID, $post);
6299 -
6300 - // Arm the double-fire guard ONLY when a post_updated is actually coming to
6301 - // consume it (plan a664f3). Two publish paths never fire post_updated at all:
6302 - // a direct wp_insert_post(status=publish) create, and wp_publish_post() — the
6303 - // call check_and_publish_future_post() makes for scheduled posts. Arming the
6304 - // guard unconditionally left it set with nothing to consume it, so the NEXT
6305 - // update of that post was swallowed entirely: zero embed calls, no knowledge
6306 - // -base row, silently. Consume-once on this side too, so a guard can never
6307 - // outlive the single save it was armed for.
6308 - if (!empty($this->pending_post_update[$post->ID])) {
6309 - unset($this->pending_post_update[$post->ID]);
6310 - $this->transition_indexed_posts[$post->ID] = true;
6311 - }
6312 - }
6313 - return;
3114 +
3115 + // Use the centralized utility function for storage
3116 + $result = MxChat_Utils::submit_content_to_db(
3117 + $final_content,
3118 + $source_url,
3119 + $api_key,
3120 + md5($source_url) // Vector ID for Pinecone
3121 + );
3122 +
3123 + if (is_wp_error($result)) {
3124 + //error_log('MxChat Auto-sync failed for post ' . $post_id . ': ' . $result->get_error_message());
6314 3125 }
6315 -
6316 - // Only the publish -> not-publish edge matters here.
6317 - if ($old_status !== 'publish' || $new_status === 'publish') {
6318 - return;
6319 - }
6320 - // Trash is handled by mxchat_handle_post_delete (wp_trash_post) with pre-trash URL
6321 - // resolution; skip to avoid a second network round-trip per trash.
6322 - if ($new_status === 'trash') {
6323 - return;
6324 - }
6325 - if (!$this->mxchat_is_auto_sync_enabled($post->post_type)) {
6326 - return;
6327 - }
6328 -
6329 - $urls = array();
6330 -
6331 - // The DB may already hold the new status when this fires, so get_permalink() on the
6332 - // live post could build a draft-style URL whose md5 misses the stored vector IDs.
6333 - // Reconstruct the published permalink from a clone instead.
6334 - $published_clone = clone $post;
6335 - $published_clone->post_status = 'publish';
6336 - $published_url = get_permalink($published_clone);
6337 - if ($published_url) {
6338 - $urls[] = $published_url;
6339 - }
6340 -
6341 - // Honour the pre-update capture when present (covers a slug change in the same save).
6342 - $previous_url = get_transient('mxchat_prev_url_' . $post->ID);
6343 - if (!empty($previous_url)) {
6344 - $urls[] = $previous_url;
6345 - }
6346 -
6347 - foreach (array_unique($urls) as $url) {
6348 - MxChat_Utils::delete_chunks_for_url($url, 'default');
6349 - }
6350 -
6351 - if (!empty($urls)) {
6352 - $this->transition_deleted_posts[$post->ID] = true;
6353 - }
6354 3126 }
6355 3127
6356 -/**
6357 - * WP-CLI: remove knowledge-base entries left behind by posts that were unpublished,
6358 - * trashed, or made private before the transition_post_status handler existed.
6359 - *
6360 - * Walks every auto-synced post type's non-published posts, reconstructs each one's
6361 - * published-era permalink, and deletes its vectors (routes to Pinecone or the WP table).
6362 - * Deletion is idempotent, so never-indexed posts are a cheap no-op.
6363 - *
6364 - * ## OPTIONS
6365 - *
6366 - * [--dry-run]
6367 - * : Report what would be removed without deleting anything.
6368 - *
6369 - * ## EXAMPLES
6370 - *
6371 - * wp mxchat prune-unpublished --dry-run
6372 - * wp mxchat prune-unpublished
6373 - */
6374 -public function cli_prune_unpublished($args, $assoc_args) {
6375 - global $wpdb;
6376 - $dry_run = !empty($assoc_args['dry-run']);
6377 - $table = $wpdb->prefix . 'mxchat_system_prompt_content';
6378 3128
6379 - $candidate_types = array_merge(array('post', 'page'), array_values(get_post_types(array('_builtin' => false), 'names')));
6380 - $synced_types = array();
6381 - foreach ($candidate_types as $type) {
6382 - if ($this->mxchat_is_auto_sync_enabled($type)) {
6383 - $synced_types[] = $type;
6384 - }
6385 - }
6386 - if (empty($synced_types)) {
6387 - WP_CLI::success('No post types have auto-sync enabled; nothing to prune.');
6388 - return;
6389 - }
6390 -
6391 - $scanned = 0;
6392 - $pruned = 0;
6393 - $paged = 1;
6394 - do {
6395 - $query = new WP_Query(array(
6396 - 'post_type' => $synced_types,
6397 - 'post_status' => array('draft', 'pending', 'private', 'future', 'trash'),
6398 - 'posts_per_page' => 100,
6399 - 'paged' => $paged,
6400 - 'fields' => 'ids',
6401 - ));
6402 - foreach ($query->posts as $post_id) {
6403 - $post = get_post($post_id);
6404 - if (!$post) {
6405 - continue;
6406 - }
6407 - $scanned++;
6408 -
6409 - // Rebuild the permalink the post had while published: publish-status clone,
6410 - // with wp_trash_post's __trashed slug suffix stripped for trashed posts.
6411 - $clone = clone $post;
6412 - $clone->post_status = 'publish';
6413 - if (substr($clone->post_name, -9) === '__trashed') {
6414 - $clone->post_name = substr($clone->post_name, 0, -9);
6415 - }
6416 - $url = get_permalink($clone);
6417 - if (!$url) {
6418 - continue;
6419 - }
6420 -
6421 - // Local-table row count is exact in WordPress-DB mode; in Pinecone mode it
6422 - // reads 0 but the delete below still routes to Pinecone and is idempotent.
6423 - $local_rows = (int) $wpdb->get_var($wpdb->prepare(
6424 - "SELECT COUNT(*) FROM {$table} WHERE source_url = %s", $url
6425 - ));
6426 -
6427 - if ($dry_run) {
6428 - if ($local_rows > 0) {
6429 - WP_CLI::log(sprintf('Would remove %d row(s): %s (post %d, %s)', $local_rows, $url, $post_id, $post->post_status));
6430 - $pruned += $local_rows;
6431 - }
6432 - continue;
6433 - }
6434 -
6435 - MxChat_Utils::delete_chunks_for_url($url, 'default');
6436 - if ($local_rows > 0) {
6437 - WP_CLI::log(sprintf('Removed %d row(s): %s (post %d, %s)', $local_rows, $url, $post_id, $post->post_status));
6438 - $pruned += $local_rows;
6439 - }
6440 - }
6441 - $more = $paged < $query->max_num_pages;
6442 - $paged++;
6443 - } while ($more);
6444 -
6445 - WP_CLI::success(sprintf(
6446 - '%s %d local knowledge row(s) across %d non-published post(s) scanned.%s',
6447 - $dry_run ? 'Would remove' : 'Removed',
6448 - $pruned,
6449 - $scanned,
6450 - ' (Pinecone-mode deletions are not counted locally.)'
6451 - ));
6452 -}
6453 -
6454 3129 public function mxchat_handle_post_delete($post_id) {
6455 3130 // Get post data before it's deleted
6456 3131 $post = get_post($post_id);
6457 3132
@@ -6481,51 +3156,85 @@
6481 3156 if (!$should_sync) {
6482 3157 return;
6483 3158 }
6484 3159
6485 - // Resolve the pre-trash URL. wp_trash_post renames the slug with "__trashed" before firing
6486 - // this hook, so get_permalink() here would return the trashed URL and md5() would miss the
6487 - // real vector IDs stored under the original URL.
6488 - $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
3160 + // Get the URL before post is deleted
3161 + $source_url = get_permalink($post_id);
6489 3162 if (!$source_url) {
6490 - //error_log('MXChat: Failed to resolve source URL for post ' . $post_id);
3163 + //error_log('MXChat: Failed to get permalink for post ' . $post_id);
6491 3164 return;
6492 3165 }
6493 3166
6494 - // Use chunk-aware deletion (handles both chunked and non-chunked content)
6495 - $delete_result = MxChat_Utils::delete_chunks_for_url($source_url, 'default');
3167 + // Check if Pinecone is enabled
3168 + $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
3169 + $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6496 3170
6497 - if (is_wp_error($delete_result)) {
6498 - //error_log('MXChat: Chunk-aware deletion failed for URL: ' . $source_url . ' - ' . $delete_result->get_error_message());
3171 + if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3172 + // Delete from Pinecone
3173 + $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
3174 + } else {
3175 + // Delete from WordPress DB
3176 + global $wpdb;
3177 + $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3178 +
3179 + $result = $wpdb->delete(
3180 + $table_name,
3181 + array('source_url' => $source_url),
3182 + array('%s')
3183 + );
3184 +
3185 + if ($result === false) {
3186 + //error_log('MXChat: WordPress DB deletion failed for URL: ' . $source_url);
3187 + }
6499 3188 }
3189 +}
6500 3190
6501 - delete_transient('mxchat_prev_url_' . $post_id);
6502 - delete_transient('mxchat_prev_status_' . $post_id);
6503 -}
6504 3191
6505 -/**
6506 - * Resolve the source URL for a post being trashed/deleted.
6507 - *
6508 - * Why: wp_trash_post appends "__trashed" to the slug before the wp_trash_post action fires, so
6509 - * get_permalink() returns a URL whose md5() won't match the vector IDs stored in Pinecone or
6510 - * the source_url rows in the WP DB. Prefer the URL captured by mxchat_store_pre_update_status
6511 - * (runs on pre_post_update, before the rename); fall back to stripping the __trashed suffix.
6512 - */
6513 -private function mxchat_resolve_pre_trash_url($post_id) {
6514 - $previous_url = get_transient('mxchat_prev_url_' . $post_id);
6515 - if (!empty($previous_url)) {
6516 - return $previous_url;
6517 - }
3192 + /**
3193 + * Deletes data from Pinecone using a source URL
3194 + */
3195 + public function mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options) {
3196 + $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
3197 + $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
6518 3198
6519 - $current = get_permalink($post_id);
6520 - if (!$current) {
6521 - return '';
6522 - }
6523 - return preg_replace('#__trashed(/?)$#', '$1', $current);
6524 -}
3199 + if (empty($host) || empty($api_key)) {
3200 + //error_log('MXChat: Pinecone deletion failed - missing configuration');
3201 + return false;
3202 + }
6525 3203
3204 + $api_endpoint = "https://{$host}/vectors/delete";
3205 + $vector_id = md5($source_url);
6526 3206
3207 + $request_body = array(
3208 + 'ids' => array($vector_id)
3209 + );
6527 3210
3211 + $response = wp_remote_post($api_endpoint, array(
3212 + 'headers' => array(
3213 + 'Api-Key' => $api_key,
3214 + 'accept' => 'application/json',
3215 + 'content-type' => 'application/json'
3216 + ),
3217 + 'body' => wp_json_encode($request_body),
3218 + 'timeout' => 30
3219 + ));
3220 +
3221 + if (is_wp_error($response)) {
3222 + //error_log('MXChat: Pinecone deletion error - ' . $response->get_error_message());
3223 + return false;
3224 + }
3225 +
3226 + $response_code = wp_remote_retrieve_response_code($response);
3227 + if ($response_code !== 200) {
3228 + //error_log('MXChat: Pinecone deletion failed with status ' . $response_code);
3229 + return false;
3230 + }
3231 +
3232 + return true;
3233 + }
3234 +
3235 +
3236 +
6528 3237 public function mxchat_handle_product_change($post_id, $post, $update) {
6529 3238 if ($post->post_type !== 'product') {
6530 3239 return;
6531 3240 }
@@ -6547,12 +3256,11 @@
6547 3256 if (!isset($this->options['enable_woocommerce_integration']) ||
6548 3257 !in_array($this->options['enable_woocommerce_integration'], ['1', 'on'])) {
6549 3258 return;
6550 3259 }
6551 -
3260 +
6552 3261 $source_url = get_permalink($product->get_id());
6553 - $product_id = $product->get_id();
6554 -
3262 +
6555 3263 // Build product content
6556 3264 $title = $product->get_name();
6557 3265 $description = $product->get_description();
6558 3266 $short_description = $product->get_short_description();
@@ -6557,97 +3265,48 @@
6557 3265 $description = $product->get_description();
6558 3266 $short_description = $product->get_short_description();
6559 3267 $regular_price = $product->get_regular_price();
6560 3268 $sale_price = $product->get_sale_price();
6561 - $price = $product->get_price();
6562 3269 $sku = $product->get_sku();
6563 -
6564 - // Get currency symbol
6565 - $currency_symbol = get_woocommerce_currency_symbol();
6566 -
3270 +
6567 3271 // Format content consistently
6568 3272 $content = $title . "\n\n";
6569 -
6570 - if (!empty($short_description)) {
6571 - $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
6572 - }
6573 -
3273 +
6574 3274 if (!empty($description)) {
6575 3275 $content .= wp_strip_all_tags($description) . "\n\n";
6576 3276 }
6577 -
6578 - // Add pricing information
6579 - if (!empty($regular_price)) {
6580 - $content .= "Price: " . $currency_symbol . $regular_price . "\n";
6581 - } elseif (!empty($price)) {
6582 - $content .= "Price: " . $currency_symbol . $price . "\n";
3277 +
3278 + if (!empty($short_description)) {
3279 + $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
6583 3280 }
6584 -
6585 - if (!empty($sale_price) && $sale_price !== $regular_price) {
6586 - $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
3281 +
3282 + $content .= "Price: $" . $regular_price . "\n";
3283 +
3284 + if (!empty($sale_price)) {
3285 + $content .= "Sale Price: $" . $sale_price . "\n";
6587 3286 }
6588 -
6589 - // Handle variable products - show price range
6590 - if ($product->is_type('variable')) {
6591 - $min_price = $product->get_variation_price('min');
6592 - $max_price = $product->get_variation_price('max');
6593 - if ($min_price !== $max_price) {
6594 - $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
6595 - }
6596 - }
6597 -
3287 +
6598 3288 if (!empty($sku)) {
6599 3289 $content .= "SKU: " . $sku . "\n";
6600 3290 }
6601 -
6602 - // Get product categories
6603 - $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
6604 - if (!empty($categories) && !is_wp_error($categories)) {
6605 - $content .= "Categories: " . implode(', ', $categories) . "\n";
3291 +
3292 + // Get API key with proper model detection
3293 + $options = get_option('mxchat_options');
3294 + $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
3295 +
3296 + if (strpos($selected_model, 'voyage') === 0) {
3297 + $api_key = $options['voyage_api_key'] ?? '';
3298 + } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
3299 + $api_key = $options['gemini_api_key'] ?? '';
3300 + } else {
3301 + $api_key = $options['api_key'] ?? '';
6606 3302 }
6607 -
6608 - // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
6609 - $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
6610 - if (!empty($custom_tabs) && is_array($custom_tabs)) {
6611 - foreach ($custom_tabs as $tab) {
6612 - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
6613 - $tab_content = isset($tab['content']) ? $tab['content'] : '';
6614 -
6615 - if (!empty($tab_title) && !empty($tab_content)) {
6616 - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
6617 - }
6618 - }
6619 - }
6620 -
6621 - // Also check for reusable/saved tabs applied to this product
6622 - $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
6623 - if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
6624 - // Get the saved tabs option
6625 - $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
6626 - if (!empty($saved_tabs) && is_array($saved_tabs)) {
6627 - foreach ($applied_saved_tabs as $saved_tab_id) {
6628 - if (isset($saved_tabs[$saved_tab_id])) {
6629 - $tab = $saved_tabs[$saved_tab_id];
6630 - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
6631 - $tab_content = isset($tab['content']) ? $tab['content'] : '';
6632 -
6633 - if (!empty($tab_title) && !empty($tab_content)) {
6634 - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
6635 - }
6636 - }
6637 - }
6638 - }
6639 - }
6640 -
6641 - // Embedding decision — custom-provider-aware (plan cbd5fd); silent-return
6642 - // shape preserved.
6643 - $preflight = MxChat_Utils::embedding_preflight(get_option('mxchat_options'));
6644 - if (!$preflight['ok']) {
6645 - //error_log('MxChat Auto-sync: embedding pre-flight failed: ' . $preflight['reason']);
3303 +
3304 + if (empty($api_key)) {
3305 + //error_log('MxChat Auto-sync: No API key configured for embedding model');
6646 3306 return;
6647 3307 }
6648 - $api_key = $preflight['api_key'];
6649 -
3308 +
6650 3309 // Use the centralized utility function for storage
6651 3310 $result = MxChat_Utils::submit_content_to_db(
6652 3311 $content,
6653 3312 $source_url,
@@ -6653,16 +3312,11 @@
6653 3312 $source_url,
6654 3313 $api_key,
6655 3314 md5($source_url) // Vector ID for Pinecone
6656 3315 );
6657 -
6658 - // After successful storage, apply role restriction based on tags
6659 - if (!is_wp_error($result)) {
6660 - $this->apply_role_restriction_to_post($product_id, $source_url);
6661 - }
6662 -
3316 +
6663 3317 if (is_wp_error($result)) {
6664 - //error_log('MxChat WooCommerce sync failed for product ' . $product_id . ': ' . $result->get_error_message());
3318 + //error_log('MxChat WooCommerce sync failed for product ' . $product->get_id() . ': ' . $result->get_error_message());
6665 3319 }
6666 3320 }
6667 3321
6668 3322 public function mxchat_handle_product_delete($post_id) {
@@ -6669,18 +3323,28 @@
6669 3323 if (get_post_type($post_id) !== 'product') {
6670 3324 return;
6671 3325 }
6672 3326
6673 - $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
6674 - if (!$source_url) {
6675 - return;
6676 - }
3327 + $source_url = get_permalink($post_id);
6677 3328
6678 - // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
6679 - MxChat_Utils::delete_chunks_for_url($source_url, 'default');
3329 + // Check if Pinecone is enabled
3330 + $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
3331 + $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6680 3332
6681 - delete_transient('mxchat_prev_url_' . $post_id);
6682 - delete_transient('mxchat_prev_status_' . $post_id);
3333 + if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
3334 + // Delete from Pinecone
3335 + $this->mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options);
3336 + } else {
3337 + // Delete from WordPress DB
3338 + global $wpdb;
3339 + $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
3340 +
3341 + $wpdb->delete(
3342 + $table_name,
3343 + array('source_url' => $source_url),
3344 + array('%s')
3345 + );
3346 + }
6683 3347 }
6684 3348
6685 3349 /**
6686 3350 * Handle individual Pinecone content deletion
@@ -6728,9 +3392,21 @@
6728 3392 $pinecone_options['mxchat_pinecone_host']
6729 3393 );
6730 3394
6731 3395 if ($result['success']) {
6732 - // No cache clearing needed since we removed caching
3396 + // Remove from ALL caches
3397 + $pinecone_manager->mxchat_remove_from_pinecone_vector_cache($vector_id);
3398 + $pinecone_manager->mxchat_remove_from_processed_content_caches($vector_id);
3399 +
3400 + // CLEAR ALL RELEVANT CACHES
3401 + delete_transient('mxchat_pinecone_recent_1k_cache');
3402 + delete_option('mxchat_pinecone_vector_ids_cache');
3403 + delete_option('mxchat_pinecone_processed_cache');
3404 + delete_option('mxchat_processed_content_cache');
3405 +
3406 + // Also force refresh for next page load
3407 + $pinecone_manager->mxchat_refresh_after_new_content($pinecone_options);
3408 +
6733 3409 set_transient('mxchat_admin_notice_success',
6734 3410 esc_html__('Entry deleted successfully from Pinecone.', 'mxchat'),
6735 3411 30
6736 3412 );
@@ -6743,11 +3419,9 @@
6743 3419
6744 3420 wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
6745 3421 exit;
6746 3422 }
6747 -/**
6748 - * Handle individual Pinecone content deletion via AJAX
6749 - */
3423 +
6750 3424 public function ajax_mxchat_delete_pinecone_prompt() {
6751 3425 // Verify nonce and permissions
6752 3426 if (!check_ajax_referer('mxchat_delete_pinecone_prompt_nonce', 'nonce', false)) {
6753 3427 wp_send_json_error('Invalid nonce');
@@ -6759,9 +3433,8 @@
6759 3433 exit;
6760 3434 }
6761 3435
6762 3436 $vector_id = isset($_POST['vector_id']) ? sanitize_text_field($_POST['vector_id']) : '';
6763 - $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6764 3437
6765 3438 if (empty($vector_id)) {
6766 3439 wp_send_json_error('Missing vector ID');
6767 3440 exit;
@@ -6766,20 +3439,19 @@
6766 3439 wp_send_json_error('Missing vector ID');
6767 3440 exit;
6768 3441 }
6769 3442
6770 - // Get bot-specific Pinecone settings
6771 - $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6772 - $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6773 -
3443 + // Get Pinecone settings
3444 + $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
6774 3445 $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6775 3446
6776 3447 if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
6777 - wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
3448 + wp_send_json_error('Pinecone is not properly configured');
6778 3449 exit;
6779 3450 }
6780 3451
6781 - // Delete from the correct Pinecone index
3452 + // Delete from Pinecone
3453 + $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6782 3454 $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
6783 3455 $vector_id,
6784 3456 $pinecone_options['mxchat_pinecone_api_key'],
6785 3457 $pinecone_options['mxchat_pinecone_host']
@@ -6785,1007 +3457,32 @@
6785 3457 $pinecone_options['mxchat_pinecone_host']
6786 3458 );
6787 3459
6788 3460 if ($result['success']) {
6789 - // No cache clearing needed since we removed caching
3461 + // Remove from ALL caches
3462 + $pinecone_manager->mxchat_remove_from_pinecone_vector_cache($vector_id);
3463 + $pinecone_manager->mxchat_remove_from_processed_content_caches($vector_id);
3464 +
3465 + // CLEAR ALL RELEVANT CACHES (ADD THESE LINES)
3466 + delete_transient('mxchat_pinecone_recent_1k_cache');
3467 + delete_option('mxchat_pinecone_vector_ids_cache');
3468 + delete_option('mxchat_pinecone_processed_cache');
3469 + delete_option('mxchat_processed_content_cache');
3470 +
3471 + // Also force refresh for next page load
3472 + $pinecone_manager->mxchat_refresh_after_new_content($pinecone_options);
3473 +
6790 3474 wp_send_json_success(array(
6791 3475 'message' => 'Entry deleted successfully from Pinecone',
6792 - 'vector_id' => $vector_id,
6793 - 'bot_id' => $bot_id
3476 + 'vector_id' => $vector_id
6794 3477 ));
6795 3478 } else {
6796 - MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete from Pinecone: ' . $result['message'], array('vector_id' => $vector_id, 'bot_id' => $bot_id));
6797 3479 wp_send_json_error('Failed to delete from Pinecone: ' . $result['message']);
6798 3480 }
6799 3481
6800 3482 exit;
6801 3483 }
6802 -
6803 -/**
6804 - * Handle deletion of all chunks for a given source URL via AJAX
6805 - * Follows the same pattern as ajax_mxchat_delete_pinecone_prompt
6806 - */
6807 -public function ajax_mxchat_delete_chunks_by_url() {
6808 - // Verify nonce and permissions
6809 - if (!check_ajax_referer('mxchat_delete_chunks_nonce', 'nonce', false)) {
6810 - wp_send_json_error('Invalid nonce');
6811 - exit;
6812 - }
6813 -
6814 - if (!current_user_can('manage_options')) {
6815 - wp_send_json_error('Unauthorized access');
6816 - exit;
6817 - }
6818 -
6819 - $source_url = isset($_POST['source_url']) ? esc_url_raw($_POST['source_url']) : '';
6820 - $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
6821 - $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
6822 -
6823 - if (empty($source_url)) {
6824 - wp_send_json_error('Missing source URL');
6825 - exit;
6826 - }
6827 -
6828 - // Generate the base vector ID from the source URL (same as how chunks are created)
6829 - $base_vector_id = md5($source_url);
6830 -
6831 - if ($data_source === 'pinecone') {
6832 - // Get bot-specific Pinecone settings (same as working delete function)
6833 - $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
6834 - $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
6835 -
6836 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
6837 -
6838 - if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
6839 - wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
6840 - exit;
6841 - }
6842 -
6843 - $api_key = $pinecone_options['mxchat_pinecone_api_key'];
6844 - $host = $pinecone_options['mxchat_pinecone_host'];
6845 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
6846 -
6847 - // Collect all vector IDs to delete
6848 - $vectors_to_delete = array();
6849 -
6850 - // Add the original single-vector ID (for non-chunked content)
6851 - $vectors_to_delete[] = $base_vector_id;
6852 -
6853 - // Use Pinecone list API to find all chunk vectors with this prefix
6854 - // NOTE: Pinecone List API is a GET request with query parameters, not POST
6855 - $prefix = $base_vector_id . '_chunk_';
6856 -
6857 - $query_params = array(
6858 - 'prefix' => $prefix,
6859 - 'limit' => 100
6860 - );
6861 -
6862 - if (!empty($namespace)) {
6863 - $query_params['namespace'] = $namespace;
6864 - }
6865 -
6866 - $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
6867 -
6868 - $list_response = wp_remote_get($list_url, array(
6869 - 'headers' => array(
6870 - 'Api-Key' => $api_key,
6871 - 'accept' => 'application/json'
6872 - ),
6873 - 'timeout' => 30
6874 - ));
6875 -
6876 - if (!is_wp_error($list_response)) {
6877 - $list_body_response = wp_remote_retrieve_body($list_response);
6878 - $list_data = json_decode($list_body_response, true);
6879 - if (!empty($list_data['vectors'])) {
6880 - foreach ($list_data['vectors'] as $vector) {
6881 - if (isset($vector['id'])) {
6882 - $vectors_to_delete[] = $vector['id'];
6883 - }
6884 - }
6885 - }
6886 - }
6887 -
6888 - if (empty($vectors_to_delete)) {
6889 - wp_send_json_success(array(
6890 - 'message' => 'No vectors found to delete',
6891 - 'source_url' => $source_url
6892 - ));
6893 - exit;
6894 - }
6895 -
6896 - // Delete all vectors using the same endpoint as the working function
6897 - $delete_url = "https://{$host}/vectors/delete";
6898 -
6899 - $delete_body = array(
6900 - 'ids' => $vectors_to_delete
6901 - );
6902 -
6903 - if (!empty($namespace)) {
6904 - $delete_body['namespace'] = $namespace;
6905 - }
6906 -
6907 - $delete_response = wp_remote_post($delete_url, array(
6908 - 'headers' => array(
6909 - 'Api-Key' => $api_key,
6910 - 'accept' => 'application/json',
6911 - 'content-type' => 'application/json'
6912 - ),
6913 - 'body' => wp_json_encode($delete_body),
6914 - 'timeout' => 30
6915 - ));
6916 -
6917 - if (is_wp_error($delete_response)) {
6918 - MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete chunks from Pinecone: ' . $delete_response->get_error_message(), array('source_url' => $source_url));
6919 - wp_send_json_error('Failed to delete from Pinecone: ' . $delete_response->get_error_message());
6920 - exit;
6921 - }
6922 -
6923 - $response_code = wp_remote_retrieve_response_code($delete_response);
6924 -
6925 - if ($response_code !== 200) {
6926 - MxChat_Admin::mxchat_log_debug('pinecone_error', 'Pinecone API error (HTTP ' . $response_code . ')', array('source_url' => $source_url));
6927 - wp_send_json_error('Pinecone API error (HTTP ' . $response_code . ')');
6928 - exit;
6929 - }
6930 -
6931 - wp_send_json_success(array(
6932 - 'message' => 'All chunks deleted successfully from Pinecone',
6933 - 'source_url' => $source_url,
6934 - 'deleted_count' => count($vectors_to_delete)
6935 - ));
6936 -
6937 - } else {
6938 - // WordPress database deletion
6939 - global $wpdb;
6940 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6941 -
6942 - $result = $wpdb->delete(
6943 - $table_name,
6944 - array('source_url' => $source_url),
6945 - array('%s')
6946 - );
6947 -
6948 - if ($result === false) {
6949 - MxChat_Admin::mxchat_log_debug('database_error', 'Failed to delete from database: ' . $wpdb->last_error, array('source_url' => $source_url));
6950 - wp_send_json_error('Failed to delete from database: ' . $wpdb->last_error);
6951 - exit;
6952 - }
6953 -
6954 - wp_send_json_success(array(
6955 - 'message' => 'All chunks deleted successfully from database',
6956 - 'source_url' => $source_url,
6957 - 'deleted_count' => $result
6958 - ));
6959 - }
6960 -
6961 - exit;
6962 -}
6963 -
6964 -/**
6965 - * Handle individual WordPress database content deletion via AJAX
6966 - * Mirrors the Pinecone delete handler but for WordPress database entries
6967 - */
6968 -public function ajax_mxchat_delete_wordpress_prompt() {
6969 - // Verify nonce and permissions
6970 - if (!check_ajax_referer('mxchat_delete_wordpress_prompt_nonce', 'nonce', false)) {
6971 - wp_send_json_error('Invalid nonce');
6972 - exit;
6973 - }
6974 -
6975 - if (!current_user_can('manage_options')) {
6976 - wp_send_json_error('Unauthorized access');
6977 - exit;
6978 - }
6979 -
6980 - $entry_id = isset($_POST['entry_id']) ? intval($_POST['entry_id']) : 0;
6981 -
6982 - if (empty($entry_id)) {
6983 - wp_send_json_error('Missing entry ID');
6984 - exit;
6985 - }
6986 -
6987 - global $wpdb;
6988 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
6989 -
6990 - // Clear cache for this entry
6991 - wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
6992 -
6993 - // Delete from database
6994 - $result = $wpdb->delete(
6995 - $table_name,
6996 - array('id' => $entry_id),
6997 - array('%d')
6998 - );
6999 -
7000 - if ($result !== false) {
7001 - wp_send_json_success(array(
7002 - 'message' => 'Entry deleted successfully',
7003 - 'entry_id' => $entry_id
7004 - ));
7005 - } else {
7006 - wp_send_json_error('Failed to delete entry from database');
7007 - }
7008 -
7009 - exit;
7010 -}
7011 -
7012 -/**
7013 - * Handle bulk deletion of knowledge entries via AJAX
7014 - * Supports both Pinecone and WordPress database entries
7015 - */
7016 -public function ajax_mxchat_bulk_delete_knowledge() {
7017 - // Verify nonce and permissions
7018 - if (!check_ajax_referer('mxchat_bulk_delete_knowledge_nonce', 'nonce', false)) {
7019 - wp_send_json_error('Invalid nonce');
7020 - exit;
7021 - }
7022 -
7023 - if (!current_user_can('manage_options')) {
7024 - wp_send_json_error('Unauthorized access');
7025 - exit;
7026 - }
7027 -
7028 - $entries = isset($_POST['entries']) ? $_POST['entries'] : array();
7029 - $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
7030 -
7031 - if (empty($entries) || !is_array($entries)) {
7032 - wp_send_json_error('No entries provided');
7033 - exit;
7034 - }
7035 -
7036 - // Extend execution time — bulk Pinecone operations can take a while
7037 - if (function_exists('set_time_limit')) {
7038 - set_time_limit(120);
7039 - }
7040 -
7041 - $success_ids = array();
7042 - $failed_ids = array();
7043 - $errors = array();
7044 -
7045 - global $wpdb;
7046 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7047 -
7048 - // Get Pinecone manager for Pinecone deletions
7049 - $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
7050 - $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
7051 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7052 -
7053 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
7054 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
7055 -
7056 - // =============================================
7057 - // PHASE 1: Collect all Pinecone vector IDs
7058 - // and separate WordPress entries
7059 - // =============================================
7060 - $pinecone_entry_ids = array(); // entry IDs that are pinecone-sourced
7061 - $wordpress_entries = array(); // entries for WordPress DB deletion
7062 - $all_vector_ids = array(); // all pinecone vector IDs to delete in one batch
7063 -
7064 - foreach ($entries as $entry) {
7065 - $entry_id = sanitize_text_field($entry['id'] ?? '');
7066 - $source = sanitize_text_field($entry['source'] ?? 'wordpress');
7067 - $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
7068 - $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
7069 -
7070 - if (empty($entry_id)) {
7071 - continue;
7072 - }
7073 -
7074 - if ($source === 'pinecone') {
7075 - if (!$use_pinecone || empty($api_key)) {
7076 - $failed_ids[] = $entry_id;
7077 - $errors[] = "Pinecone not configured for entry: $entry_id";
7078 - continue;
7079 - }
7080 -
7081 - $pinecone_entry_ids[] = $entry_id;
7082 -
7083 - if ($is_group && !empty($source_url)) {
7084 - // Grouped/chunked entry: collect base ID + chunk IDs via List API
7085 - $base_vector_id = md5($source_url);
7086 - $all_vector_ids[] = $base_vector_id;
7087 -
7088 - $list_url = 'https://' . $host . '/vectors/list?prefix=' . urlencode($base_vector_id . '_chunk_') . '&limit=100';
7089 - $list_response = wp_remote_get($list_url, array(
7090 - 'headers' => array(
7091 - 'Api-Key' => $api_key,
7092 - 'accept' => 'application/json'
7093 - ),
7094 - 'timeout' => 30
7095 - ));
7096 -
7097 - if (!is_wp_error($list_response)) {
7098 - $list_body = json_decode(wp_remote_retrieve_body($list_response), true);
7099 - if (!empty($list_body['vectors']) && is_array($list_body['vectors'])) {
7100 - foreach ($list_body['vectors'] as $vector) {
7101 - if (isset($vector['id'])) {
7102 - $all_vector_ids[] = $vector['id'];
7103 - }
7104 - }
7105 - }
7106 - }
7107 - } else {
7108 - // Single entry: the entry_id IS the vector ID
7109 - $all_vector_ids[] = $entry_id;
7110 - }
7111 - } else {
7112 - $wordpress_entries[] = $entry;
7113 - }
7114 - }
7115 -
7116 - // =============================================
7117 - // PHASE 2: Single batch delete to Pinecone
7118 - // =============================================
7119 - if (!empty($all_vector_ids)) {
7120 - $all_vector_ids = array_values(array_unique($all_vector_ids));
7121 - $pinecone_success = true;
7122 - $batches = array_chunk($all_vector_ids, 100);
7123 -
7124 - foreach ($batches as $batch) {
7125 - $delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
7126 - 'headers' => array(
7127 - 'Api-Key' => $api_key,
7128 - 'accept' => 'application/json',
7129 - 'content-type' => 'application/json'
7130 - ),
7131 - 'body' => wp_json_encode(array('ids' => $batch)),
7132 - 'timeout' => 60
7133 - ));
7134 -
7135 - if (is_wp_error($delete_response)) {
7136 - $pinecone_success = false;
7137 - $errors[] = 'Pinecone batch deletion failed: ' . $delete_response->get_error_message();
7138 - MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed: ' . $delete_response->get_error_message());
7139 - } else {
7140 - $response_code = wp_remote_retrieve_response_code($delete_response);
7141 - if ($response_code !== 200) {
7142 - $pinecone_success = false;
7143 - $response_body = wp_remote_retrieve_body($delete_response);
7144 - $errors[] = "Pinecone API error (HTTP $response_code)";
7145 - MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed (HTTP ' . $response_code . ')', array('response' => substr($response_body, 0, 200)));
7146 - }
7147 - }
7148 - }
7149 -
7150 - // Mark all pinecone entries based on batch result
7151 - foreach ($pinecone_entry_ids as $eid) {
7152 - if ($pinecone_success) {
7153 - $success_ids[] = $eid;
7154 - } else {
7155 - $failed_ids[] = $eid;
7156 - }
7157 - }
7158 - }
7159 -
7160 - // =============================================
7161 - // PHASE 3: WordPress database deletions
7162 - // =============================================
7163 - foreach ($wordpress_entries as $entry) {
7164 - $entry_id = sanitize_text_field($entry['id'] ?? '');
7165 - $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
7166 - $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');
7167 -
7168 - if (empty($entry_id)) {
7169 - continue;
7170 - }
7171 -
7172 - try {
7173 - if ($is_group && !empty($source_url)) {
7174 - $result = $wpdb->delete(
7175 - $table_name,
7176 - array('source_url' => $source_url),
7177 - array('%s')
7178 - );
7179 - } else {
7180 - wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
7181 - $result = $wpdb->delete(
7182 - $table_name,
7183 - array('id' => intval($entry_id)),
7184 - array('%d')
7185 - );
7186 - }
7187 -
7188 - if ($result !== false) {
7189 - $success_ids[] = $entry_id;
7190 - } else {
7191 - $failed_ids[] = $entry_id;
7192 - $errors[] = "Database error for entry: $entry_id";
7193 - }
7194 - } catch (Exception $e) {
7195 - $failed_ids[] = $entry_id;
7196 - $errors[] = $e->getMessage();
7197 - }
7198 - }
7199 -
7200 - wp_send_json_success(array(
7201 - 'success_ids' => $success_ids,
7202 - 'failed_ids' => $failed_ids,
7203 - 'errors' => $errors,
7204 - 'total_processed' => count($success_ids) + count($failed_ids)
7205 - ));
7206 -
7207 - exit;
7208 -}
7209 -
7210 -/**
7211 - * Get hierarchical roles for dropdown
7212 - */
7213 -public function mxchat_get_role_options() {
7214 - return array(
7215 - 'public' => __('Public (Everyone)', 'mxchat'),
7216 - 'logged_in' => __('Logged In Users', 'mxchat'),
7217 - 'subscriber' => __('Subscribers & Above', 'mxchat'),
7218 - 'contributor' => __('Contributors & Above', 'mxchat'),
7219 - 'author' => __('Authors & Above', 'mxchat'),
7220 - 'editor' => __('Editors & Above', 'mxchat'),
7221 - 'administrator' => __('Administrators Only', 'mxchat')
7222 - );
7223 -}
7224 -
7225 -/**
7226 - * Check if user has access to content based on role restriction
7227 - */
7228 -public function mxchat_user_has_content_access($role_restriction) {
7229 - // Public content is always accessible
7230 - if ($role_restriction === 'public' || empty($role_restriction)) {
7231 - return true;
7232 - }
7233 3484
7234 - // Check if user is logged in for logged_in restriction
7235 - if ($role_restriction === 'logged_in') {
7236 - return is_user_logged_in();
7237 - }
7238 -
7239 - // If not logged in, no access to role-restricted content
7240 - if (!is_user_logged_in()) {
7241 - return false;
7242 - }
7243 -
7244 - $user = wp_get_current_user();
7245 - $user_roles = $user->roles;
7246 -
7247 - if (empty($user_roles)) {
7248 - return false;
7249 - }
7250 -
7251 - // Define role hierarchy (higher number = higher access)
7252 - $hierarchy = array(
7253 - 'subscriber' => 1,
7254 - 'contributor' => 2,
7255 - 'author' => 3,
7256 - 'editor' => 4,
7257 - 'administrator' => 5
7258 - );
7259 -
7260 - // Get required level
7261 - $required_level = isset($hierarchy[$role_restriction]) ? $hierarchy[$role_restriction] : 0;
7262 -
7263 - // Check if user has required level or higher
7264 - foreach ($user_roles as $user_role) {
7265 - $user_level = isset($hierarchy[$user_role]) ? $hierarchy[$user_role] : 0;
7266 - if ($user_level >= $required_level) {
7267 - return true;
7268 - }
7269 - }
7270 -
7271 - return false;
7272 -}
7273 -
7274 -/**
7275 - * Handle role restriction updates via AJAX
7276 - * Removed cache clearing call since we removed caching
7277 - */
7278 -public function ajax_mxchat_update_role_restriction() {
7279 - // Verify nonce and permissions
7280 - if (!check_ajax_referer('mxchat_update_role_nonce', 'nonce', false)) {
7281 - wp_send_json_error('Invalid nonce');
7282 - exit;
7283 - }
7284 -
7285 - if (!current_user_can('manage_options')) {
7286 - wp_send_json_error('Unauthorized access');
7287 - exit;
7288 - }
7289 -
7290 - $entry_id = isset($_POST['entry_id']) ? sanitize_text_field($_POST['entry_id']) : '';
7291 - $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
7292 - $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
7293 -
7294 - if (empty($entry_id)) {
7295 - wp_send_json_error('Invalid entry ID');
7296 - exit;
7297 - }
7298 -
7299 - // Get knowledge manager instance to validate role restriction
7300 - $knowledge_manager = MxChat_Knowledge_Manager::get_instance();
7301 - $valid_roles = array_keys($knowledge_manager->mxchat_get_role_options());
7302 - if (!in_array($role_restriction, $valid_roles)) {
7303 - wp_send_json_error('Invalid role restriction');
7304 - exit;
7305 - }
7306 -
7307 - global $wpdb;
7308 -
7309 - if ($data_source === 'pinecone') {
7310 - // Handle Pinecone role restriction (stored separately in WordPress table)
7311 - $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7312 -
7313 - // Use REPLACE to insert or update the role restriction
7314 - $result = $wpdb->replace(
7315 - $roles_table,
7316 - array(
7317 - 'vector_id' => $entry_id,
7318 - 'role_restriction' => $role_restriction,
7319 - 'updated_at' => current_time('mysql')
7320 - ),
7321 - array('%s', '%s', '%s')
7322 - );
7323 -
7324 - // No cache clearing needed since we removed caching
7325 -
7326 - } else {
7327 - // Handle WordPress database role restriction (existing functionality)
7328 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7329 -
7330 - $result = $wpdb->update(
7331 - $table_name,
7332 - array('role_restriction' => $role_restriction),
7333 - array('id' => absint($entry_id)),
7334 - array('%s'),
7335 - array('%d')
7336 - );
7337 - }
7338 -
7339 - if ($result === false) {
7340 - wp_send_json_error('Database update failed: ' . $wpdb->last_error);
7341 - exit;
7342 - }
7343 -
7344 - wp_send_json_success(array(
7345 - 'message' => 'Role restriction updated successfully',
7346 - 'role_restriction' => $role_restriction,
7347 - 'data_source' => $data_source,
7348 - 'entry_id' => $entry_id
7349 - ));
7350 - exit;
7351 -}
7352 -
7353 -// ========================================
7354 -// ROLE-BASED CONTENT RESTRICTIONS - NEW FUNCTIONS
7355 -// Add these to your MxChat_Knowledge_Manager class
7356 -// ========================================
7357 -
7358 -/**
7359 - * Initialize role-based content hooks
7360 - * Add this call to your __construct() or mxchat_init_hooks() method
7361 - */
7362 -private function mxchat_init_role_hooks() {
7363 - // AJAX handlers for tag-role mappings
7364 - add_action('wp_ajax_mxchat_add_tag_role_mapping', array($this, 'ajax_add_tag_role_mapping'));
7365 - add_action('wp_ajax_mxchat_delete_tag_role_mapping', array($this, 'ajax_delete_tag_role_mapping'));
7366 - add_action('wp_ajax_mxchat_get_tag_role_mappings', array($this, 'ajax_get_tag_role_mappings'));
7367 - add_action('wp_ajax_mxchat_bulk_update_tag_roles', array($this, 'ajax_bulk_update_tag_roles'));
7368 -
7369 - // Hook to automatically update role restrictions when tags are added/removed
7370 - add_action('set_object_terms', array($this, 'handle_tag_change'), 10, 6);
7371 -
7372 - // Hook to apply role restrictions on auto-sync
7373 - add_action('mxchat_content_stored', array($this, 'apply_role_restriction_after_storage'), 10, 2);
7374 -}
7375 -
7376 -/**
7377 - * Add tag-role mapping via AJAX
7378 - */
7379 -public function ajax_add_tag_role_mapping() {
7380 - // Verify nonce and permissions
7381 - check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7382 -
7383 - if (!current_user_can('manage_options')) {
7384 - wp_send_json_error('Unauthorized access');
7385 - exit;
7386 - }
7387 -
7388 - $tag_input = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
7389 - $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
7390 -
7391 - if (empty($tag_input)) {
7392 - wp_send_json_error('Please enter a tag name or slug');
7393 - exit;
7394 - }
7395 -
7396 - // Validate role restriction
7397 - $valid_roles = array_keys($this->mxchat_get_role_options());
7398 - if (!in_array($role_restriction, $valid_roles)) {
7399 - wp_send_json_error('Invalid role restriction');
7400 - exit;
7401 - }
7402 -
7403 - // Resolve the tag by slug first, then fall back to its display name, so users can
7404 - // enter either "premium-content" or "Premium Content". (plan b8bcf5 — the field is
7405 - // labeled by name but previously validated by slug only, producing the confusing
7406 - // "Tag does not exist in WordPress" error when a real tag's name was typed.)
7407 - $term = get_term_by('slug', $tag_input, 'post_tag');
7408 - if (!$term) {
7409 - $term = get_term_by('name', $tag_input, 'post_tag');
7410 - }
7411 - if (!$term) {
7412 - wp_send_json_error('No tag with that name or slug exists yet. Create it under Posts → Tags first, then enter its name or slug.');
7413 - exit;
7414 - }
7415 -
7416 - // Always key the mapping by the RESOLVED slug — apply_role_restriction_to_post()
7417 - // compares against each post's tag slugs, so the stored key must be a slug,
7418 - // never the raw (possibly display-name) input.
7419 - $tag_slug = $term->slug;
7420 -
7421 - // Get existing mappings
7422 - $mappings = get_option('mxchat_tag_role_mappings', array());
7423 -
7424 - // Check if mapping already exists
7425 - if (isset($mappings[$tag_slug])) {
7426 - wp_send_json_error('Mapping for this tag already exists');
7427 - exit;
7428 - }
7429 -
7430 - // Add new mapping
7431 - $mappings[$tag_slug] = $role_restriction;
7432 - update_option('mxchat_tag_role_mappings', $mappings);
7433 -
7434 - wp_send_json_success(array(
7435 - 'message' => 'Tag-role mapping added successfully',
7436 - 'tag_slug' => $tag_slug,
7437 - 'role_restriction' => $role_restriction
7438 - ));
7439 - exit;
7440 -}
7441 -
7442 -/**
7443 - * Delete tag-role mapping via AJAX
7444 - */
7445 -public function ajax_delete_tag_role_mapping() {
7446 - // Verify nonce and permissions
7447 - check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7448 -
7449 - if (!current_user_can('manage_options')) {
7450 - wp_send_json_error('Unauthorized access');
7451 - exit;
7452 - }
7453 -
7454 - $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
7455 -
7456 - if (empty($tag_slug)) {
7457 - wp_send_json_error('Tag slug is required');
7458 - exit;
7459 - }
7460 -
7461 - // Get existing mappings
7462 - $mappings = get_option('mxchat_tag_role_mappings', array());
7463 -
7464 - // Check if mapping exists
7465 - if (!isset($mappings[$tag_slug])) {
7466 - wp_send_json_error('Mapping does not exist');
7467 - exit;
7468 - }
7469 -
7470 - // Remove mapping
7471 - unset($mappings[$tag_slug]);
7472 - update_option('mxchat_tag_role_mappings', $mappings);
7473 -
7474 - wp_send_json_success(array(
7475 - 'message' => 'Tag-role mapping deleted successfully',
7476 - 'tag_slug' => $tag_slug
7477 - ));
7478 - exit;
7479 -}
7480 -
7481 -/**
7482 - * Get all tag-role mappings via AJAX
7483 - */
7484 -public function ajax_get_tag_role_mappings() {
7485 - // Verify nonce and permissions
7486 - check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7487 -
7488 - if (!current_user_can('manage_options')) {
7489 - wp_send_json_error('Unauthorized access');
7490 - exit;
7491 - }
7492 -
7493 - // Get mappings
7494 - $mappings = get_option('mxchat_tag_role_mappings', array());
7495 - $role_options = $this->mxchat_get_role_options();
7496 -
7497 - $formatted_mappings = array();
7498 -
7499 - foreach ($mappings as $tag_slug => $role_restriction) {
7500 - // Get tag object
7501 - $term = get_term_by('slug', $tag_slug, 'post_tag');
7502 -
7503 - // Count posts with this tag
7504 - $post_count = 0;
7505 - if ($term) {
7506 - $post_count = $term->count;
7507 - }
7508 -
7509 - $formatted_mappings[] = array(
7510 - 'tag_slug' => $tag_slug,
7511 - 'role_restriction' => $role_restriction,
7512 - 'role_label' => $role_options[$role_restriction] ?? $role_restriction,
7513 - 'post_count' => $post_count
7514 - );
7515 - }
7516 -
7517 - wp_send_json_success(array(
7518 - 'mappings' => $formatted_mappings
7519 - ));
7520 - exit;
7521 -}
7522 -
7523 -/**
7524 - * Bulk update role restrictions for all existing content with mapped tags
7525 - */
7526 -public function ajax_bulk_update_tag_roles() {
7527 - // Verify nonce and permissions
7528 - check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
7529 -
7530 - if (!current_user_can('manage_options')) {
7531 - wp_send_json_error('Unauthorized access');
7532 - exit;
7533 - }
7534 -
7535 - // Get mappings
7536 - $mappings = get_option('mxchat_tag_role_mappings', array());
7537 -
7538 - if (empty($mappings)) {
7539 - wp_send_json_error('No tag-role mappings found');
7540 - exit;
7541 - }
7542 -
7543 - global $wpdb;
7544 -
7545 - // Check if using Pinecone
7546 - $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
7547 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7548 -
7549 - $updated_count = 0;
7550 - $details = array();
7551 -
7552 - foreach ($mappings as $tag_slug => $role_restriction) {
7553 - // Get all posts with this tag
7554 - $posts = get_posts(array(
7555 - 'tag' => $tag_slug,
7556 - 'post_type' => 'any',
7557 - 'posts_per_page' => -1,
7558 - 'fields' => 'ids',
7559 - 'post_status' => 'publish'
7560 - ));
7561 -
7562 - if (empty($posts)) {
7563 - continue;
7564 - }
7565 -
7566 - $tag_updated = 0;
7567 -
7568 - foreach ($posts as $post_id) {
7569 - $source_url = get_permalink($post_id);
7570 - if (!$source_url) {
7571 - continue;
7572 - }
7573 -
7574 - if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
7575 - // Update Pinecone role restriction
7576 - $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7577 - $vector_id = md5($source_url);
7578 -
7579 - $result = $wpdb->replace(
7580 - $roles_table,
7581 - array(
7582 - 'vector_id' => $vector_id,
7583 - 'role_restriction' => $role_restriction,
7584 - 'updated_at' => current_time('mysql')
7585 - ),
7586 - array('%s', '%s', '%s')
7587 - );
7588 - } else {
7589 - // Update WordPress DB
7590 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7591 -
7592 - $result = $wpdb->update(
7593 - $table_name,
7594 - array('role_restriction' => $role_restriction),
7595 - array('source_url' => $source_url),
7596 - array('%s'),
7597 - array('%s')
7598 - );
7599 - }
7600 -
7601 - if ($result !== false) {
7602 - $tag_updated++;
7603 - $updated_count++;
7604 - }
7605 - }
7606 -
7607 - if ($tag_updated > 0) {
7608 - $details[] = sprintf(
7609 - 'Tag "%s" (%s): %d posts updated',
7610 - $tag_slug,
7611 - $role_restriction,
7612 - $tag_updated
7613 - );
7614 - }
7615 - }
7616 -
7617 - wp_send_json_success(array(
7618 - 'message' => 'Bulk update completed',
7619 - 'updated_count' => $updated_count,
7620 - 'tags_processed' => count($mappings),
7621 - 'details' => $details
7622 - ));
7623 - exit;
7624 -}
7625 -
7626 -/**
7627 - * Handle tag changes on posts (when tags are added or removed)
7628 - */
7629 -public function handle_tag_change($object_id, $terms, $tt_ids, $taxonomy, $append, $old_tt_ids) {
7630 - // Only process post tags
7631 - if ($taxonomy !== 'post_tag') {
7632 - return;
7633 - }
7634 -
7635 - // Get tag-role mappings
7636 - $mappings = get_option('mxchat_tag_role_mappings', array());
7637 -
7638 - if (empty($mappings)) {
7639 - return;
7640 - }
7641 -
7642 - // Get the post's URL
7643 - $source_url = get_permalink($object_id);
7644 - if (!$source_url) {
7645 - return;
7646 - }
7647 -
7648 - // Determine the highest role restriction based on tags
7649 - $highest_role = 'public';
7650 - $role_hierarchy = array(
7651 - 'public' => 0,
7652 - 'logged_in' => 1,
7653 - 'subscriber' => 2,
7654 - 'contributor' => 3,
7655 - 'author' => 4,
7656 - 'editor' => 5,
7657 - 'administrator' => 6
7658 - );
7659 -
7660 - // Get all current tags for the post
7661 - $current_tags = wp_get_post_tags($object_id, array('fields' => 'slugs'));
7662 -
7663 - // Find the highest role restriction among the tags
7664 - foreach ($current_tags as $tag_slug) {
7665 - if (isset($mappings[$tag_slug])) {
7666 - $role = $mappings[$tag_slug];
7667 - if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
7668 - $highest_role = $role;
7669 - }
7670 - }
7671 - }
7672 -
7673 - // Update the role restriction in the database
7674 - global $wpdb;
7675 -
7676 - // Check if using Pinecone
7677 - $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
7678 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7679 -
7680 - if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
7681 - // Update Pinecone role restriction
7682 - $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7683 - $vector_id = md5($source_url);
7684 -
7685 - $wpdb->replace(
7686 - $roles_table,
7687 - array(
7688 - 'vector_id' => $vector_id,
7689 - 'role_restriction' => $highest_role,
7690 - 'updated_at' => current_time('mysql')
7691 - ),
7692 - array('%s', '%s', '%s')
7693 - );
7694 - } else {
7695 - // Update WordPress DB
7696 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7697 -
7698 - $wpdb->update(
7699 - $table_name,
7700 - array('role_restriction' => $highest_role),
7701 - array('source_url' => $source_url),
7702 - array('%s'),
7703 - array('%s')
7704 - );
7705 - }
7706 -}
7707 -
7708 -/**
7709 - * Apply role restriction after content is stored (for auto-sync)
7710 - */
7711 -public function apply_role_restriction_after_storage($post_id, $source_url) {
7712 - // Get tag-role mappings
7713 - $mappings = get_option('mxchat_tag_role_mappings', array());
7714 -
7715 - if (empty($mappings)) {
7716 - return;
7717 - }
7718 -
7719 - // Get all tags for the post
7720 - $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
7721 -
7722 - if (empty($post_tags)) {
7723 - return;
7724 - }
7725 -
7726 - // Determine the highest role restriction based on tags
7727 - $highest_role = 'public';
7728 - $role_hierarchy = array(
7729 - 'public' => 0,
7730 - 'logged_in' => 1,
7731 - 'subscriber' => 2,
7732 - 'contributor' => 3,
7733 - 'author' => 4,
7734 - 'editor' => 5,
7735 - 'administrator' => 6
7736 - );
7737 -
7738 - foreach ($post_tags as $tag_slug) {
7739 - if (isset($mappings[$tag_slug])) {
7740 - $role = $mappings[$tag_slug];
7741 - if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
7742 - $highest_role = $role;
7743 - }
7744 - }
7745 - }
7746 -
7747 - // If no restricted tags found, return (leave as public)
7748 - if ($highest_role === 'public') {
7749 - return;
7750 - }
7751 -
7752 - // Update the role restriction
7753 - global $wpdb;
7754 -
7755 - // Check if using Pinecone
7756 - $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
7757 - $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
7758 -
7759 - if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
7760 - // Update Pinecone role restriction
7761 - $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
7762 - $vector_id = md5($source_url);
7763 -
7764 - $wpdb->replace(
7765 - $roles_table,
7766 - array(
7767 - 'vector_id' => $vector_id,
7768 - 'role_restriction' => $highest_role,
7769 - 'updated_at' => current_time('mysql')
7770 - ),
7771 - array('%s', '%s', '%s')
7772 - );
7773 - } else {
7774 - // Update WordPress DB
7775 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
7776 -
7777 - $wpdb->update(
7778 - $table_name,
7779 - array('role_restriction' => $highest_role),
7780 - array('source_url' => $source_url),
7781 - array('%s'),
7782 - array('%s')
7783 - );
7784 - }
7785 -}
7786 -
7787 -
7788 3485 // ========================================
7789 3486 // HELPER METHODS
7790 3487 // ========================================
7791 3488
@@ -7852,1062 +3549,8 @@
7852 3549 */
7853 3550 private function mxchat_get_pinecone_manager() {
7854 3551 return MxChat_Pinecone_Manager::get_instance();
7855 3552 }
7856 -
7857 -
7858 - // ========================================
7859 -// DATABASE QUEUE TABLE MANAGEMENT
7860 -// ========================================
7861 -
7862 -/**
7863 - * Create queue table on plugin activation
7864 - * Call this from your plugin activation hook
7865 - */
7866 -public function mxchat_create_queue_table() {
7867 - global $wpdb;
7868 -
7869 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7870 - $charset_collate = $wpdb->get_charset_collate();
7871 -
7872 - $sql = "CREATE TABLE IF NOT EXISTS $table_name (
7873 - id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
7874 - queue_id varchar(64) NOT NULL,
7875 - item_type varchar(20) NOT NULL,
7876 - item_data longtext NOT NULL,
7877 - status varchar(20) NOT NULL DEFAULT 'pending',
7878 - bot_id varchar(50) NOT NULL DEFAULT 'default',
7879 - priority int(11) NOT NULL DEFAULT 0,
7880 - attempts int(11) NOT NULL DEFAULT 0,
7881 - max_attempts int(11) NOT NULL DEFAULT 3,
7882 - error_message text DEFAULT NULL,
7883 - created_at datetime NOT NULL,
7884 - started_at datetime DEFAULT NULL,
7885 - completed_at datetime DEFAULT NULL,
7886 - PRIMARY KEY (id),
7887 - KEY queue_id (queue_id),
7888 - KEY status (status),
7889 - KEY item_type (item_type),
7890 - KEY priority (priority)
7891 - ) $charset_collate;";
7892 -
7893 - require_once(ABSPATH . 'wp-admin/includes/upgrade.php');
7894 - dbDelta($sql);
7895 -
7896 - // Also create a meta table for queue metadata
7897 - $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7898 -
7899 - $meta_sql = "CREATE TABLE IF NOT EXISTS $meta_table (
7900 - id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
7901 - queue_id varchar(64) NOT NULL,
7902 - meta_key varchar(255) NOT NULL,
7903 - meta_value longtext,
7904 - PRIMARY KEY (id),
7905 - KEY queue_id (queue_id),
7906 - KEY meta_key (meta_key)
7907 - ) $charset_collate;";
7908 -
7909 - dbDelta($meta_sql);
7910 -}
7911 -
7912 -/**
7913 - * Add items to the processing queue
7914 - *
7915 - * @param string $queue_id Unique identifier for this queue batch
7916 - * @param string $item_type Type of item (url, pdf_page)
7917 - * @param array $items Array of items to queue
7918 - * @param string $bot_id Bot ID for processing
7919 - * @return int Number of items queued
7920 - */
7921 -private function mxchat_add_to_queue($queue_id, $item_type, $items, $bot_id = 'default') {
7922 - global $wpdb;
7923 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
7924 -
7925 - $queued_count = 0;
7926 - $priority = 0;
7927 -
7928 - foreach ($items as $item) {
7929 - $result = $wpdb->insert(
7930 - $table_name,
7931 - array(
7932 - 'queue_id' => $queue_id,
7933 - 'item_type' => $item_type,
7934 - 'item_data' => wp_json_encode($item),
7935 - 'status' => 'pending',
7936 - 'bot_id' => $bot_id,
7937 - 'priority' => $priority,
7938 - 'attempts' => 0,
7939 - 'max_attempts' => 3,
7940 - 'created_at' => current_time('mysql')
7941 - ),
7942 - array('%s', '%s', '%s', '%s', '%s', '%d', '%d', '%d', '%s')
7943 - );
7944 -
7945 - if ($result) {
7946 - $queued_count++;
7947 - }
7948 -
7949 - $priority++; // Process in order
7950 - }
7951 -
7952 - return $queued_count;
7953 -}
7954 -
7955 -/**
7956 - * Store queue metadata (total counts, source URL, etc.)
7957 - */
7958 -private function mxchat_set_queue_meta($queue_id, $meta_key, $meta_value) {
7959 - global $wpdb;
7960 - $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7961 -
7962 - // Check if meta exists
7963 - $existing = $wpdb->get_var($wpdb->prepare(
7964 - "SELECT id FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
7965 - $queue_id,
7966 - $meta_key
7967 - ));
7968 -
7969 - if ($existing) {
7970 - // Update
7971 - $wpdb->update(
7972 - $meta_table,
7973 - array('meta_value' => maybe_serialize($meta_value)),
7974 - array('queue_id' => $queue_id, 'meta_key' => $meta_key),
7975 - array('%s'),
7976 - array('%s', '%s')
7977 - );
7978 - } else {
7979 - // Insert
7980 - $wpdb->insert(
7981 - $meta_table,
7982 - array(
7983 - 'queue_id' => $queue_id,
7984 - 'meta_key' => $meta_key,
7985 - 'meta_value' => maybe_serialize($meta_value)
7986 - ),
7987 - array('%s', '%s', '%s')
7988 - );
7989 - }
7990 -}
7991 -
7992 -/**
7993 - * Get queue metadata
7994 - */
7995 -private function mxchat_get_queue_meta($queue_id, $meta_key) {
7996 - global $wpdb;
7997 - $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
7998 -
7999 - $value = $wpdb->get_var($wpdb->prepare(
8000 - "SELECT meta_value FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
8001 - $queue_id,
8002 - $meta_key
8003 - ));
8004 -
8005 - return maybe_unserialize($value);
8006 -}
8007 -
8008 -// ========================================
8009 -// AJAX QUEUE PROCESSING HANDLERS
8010 -// ========================================
8011 -
8012 -/**
8013 - * AJAX: Get next item from queue to process
8014 - */
8015 -public function ajax_mxchat_get_next_queue_item() {
8016 - // Verify nonce and permissions
8017 - check_ajax_referer('mxchat_queue_nonce', 'nonce');
8018 -
8019 - if (!current_user_can('manage_options')) {
8020 - wp_send_json_error('Unauthorized access');
8021 - }
8022 -
8023 - $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8024 -
8025 - if (empty($queue_id)) {
8026 - wp_send_json_error('Missing queue ID');
8027 - }
8028 -
8029 - global $wpdb;
8030 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8031 -
8032 - // Get next pending item with retry logic for failed items
8033 - $next_item = $wpdb->get_row($wpdb->prepare(
8034 - "SELECT * FROM $table_name
8035 - WHERE queue_id = %s
8036 - AND status IN ('pending', 'failed')
8037 - AND attempts < max_attempts
8038 - ORDER BY priority ASC, id ASC
8039 - LIMIT 1",
8040 - $queue_id
8041 - ));
8042 -
8043 - if (!$next_item) {
8044 - // No more items - queue complete
8045 - wp_send_json_success(array(
8046 - 'complete' => true,
8047 - 'message' => 'Queue processing complete'
8048 - ));
8049 - }
8050 -
8051 - // Mark item as processing
8052 - $wpdb->update(
8053 - $table_name,
8054 - array(
8055 - 'status' => 'processing',
8056 - 'started_at' => current_time('mysql'),
8057 - 'attempts' => $next_item->attempts + 1
8058 - ),
8059 - array('id' => $next_item->id),
8060 - array('%s', '%s', '%d'),
8061 - array('%d')
8062 - );
8063 -
8064 - wp_send_json_success(array(
8065 - 'complete' => false,
8066 - 'item' => array(
8067 - 'id' => $next_item->id,
8068 - 'type' => $next_item->item_type,
8069 - 'data' => json_decode($next_item->item_data, true),
8070 - 'bot_id' => $next_item->bot_id,
8071 - 'attempt' => $next_item->attempts + 1
8072 - )
8073 - ));
8074 -}
8075 -
8076 -/**
8077 - * AJAX: Process a single queue item
8078 - */
8079 -public function ajax_mxchat_process_queue_item() {
8080 - // Verify nonce and permissions
8081 - check_ajax_referer('mxchat_queue_nonce', 'nonce');
8082 -
8083 - if (!current_user_can('manage_options')) {
8084 - wp_send_json_error('Unauthorized access');
8085 - }
8086 -
8087 - $item_id = isset($_POST['item_id']) ? absint($_POST['item_id']) : 0;
8088 - $item_type = isset($_POST['item_type']) ? sanitize_text_field($_POST['item_type']) : '';
8089 - $item_data = isset($_POST['item_data']) ? $_POST['item_data'] : array();
8090 - $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
8091 -
8092 - if (empty($item_id) || empty($item_type)) {
8093 - wp_send_json_error('Missing item data');
8094 - }
8095 -
8096 - global $wpdb;
8097 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8098 -
8099 - // Process based on item type
8100 - try {
8101 - set_time_limit(60); // Give processing 60 seconds
8102 -
8103 - $result = false;
8104 - $error_message = '';
8105 -
8106 - // Read item directly from DB to get queue_id and preserve special chars in item_data
8107 - // (POST round-trip through JS mangles characters like apostrophes in URLs)
8108 - $db_item = $wpdb->get_row($wpdb->prepare(
8109 - "SELECT queue_id, item_data FROM $table_name WHERE id = %d",
8110 - $item_id
8111 - ));
8112 - $item_queue_id = $db_item ? $db_item->queue_id : '';
8113 - if ($db_item && !empty($db_item->item_data)) {
8114 - $db_data = json_decode($db_item->item_data, true);
8115 - if (is_array($db_data)) {
8116 - $item_data = $db_data;
8117 - }
8118 - }
8119 -
8120 - switch ($item_type) {
8121 - case 'url':
8122 - $result = $this->mxchat_process_queue_url($item_data, $bot_id, $item_queue_id);
8123 - break;
8124 -
8125 - case 'pdf_page':
8126 - $result = $this->mxchat_process_queue_pdf_page($item_data, $bot_id);
8127 - break;
8128 -
8129 - default:
8130 - throw new Exception('Unknown item type: ' . $item_type);
8131 - }
8132 -
8133 - if (is_wp_error($result)) {
8134 - $error_code = $result->get_error_code();
8135 - // Content errors (empty page, sanitization) are permanent — retrying won't help
8136 - $permanent_codes = array('empty_page', 'empty_after_sanitization', 'no_api_key', 'page_not_found');
8137 - if (in_array($error_code, $permanent_codes)) {
8138 - // Mark as permanently failed — set attempts = max_attempts so it won't be retried
8139 - $current_item = $wpdb->get_row($wpdb->prepare(
8140 - "SELECT max_attempts FROM $table_name WHERE id = %d", $item_id
8141 - ));
8142 - $wpdb->update(
8143 - $table_name,
8144 - array(
8145 - 'status' => 'failed',
8146 - 'error_message' => $result->get_error_message(),
8147 - 'attempts' => $current_item ? $current_item->max_attempts : 3
8148 - ),
8149 - array('id' => $item_id),
8150 - array('%s', '%s', '%d'),
8151 - array('%d')
8152 - );
8153 - wp_send_json_error(array(
8154 - 'message' => $result->get_error_message(),
8155 - 'permanent_failure' => true,
8156 - 'item_id' => $item_id
8157 - ));
8158 - return;
8159 - }
8160 - throw new Exception($result->get_error_message());
8161 - }
8162 -
8163 - if ($result === false) {
8164 - throw new Exception('Processing returned false - item may be empty or invalid');
8165 - }
8166 -
8167 - // Mark as completed
8168 - $wpdb->update(
8169 - $table_name,
8170 - array(
8171 - 'status' => 'completed',
8172 - 'completed_at' => current_time('mysql'),
8173 - 'error_message' => null
8174 - ),
8175 - array('id' => $item_id),
8176 - array('%s', '%s', '%s'),
8177 - array('%d')
8178 - );
8179 -
8180 - wp_send_json_success(array(
8181 - 'processed' => true,
8182 - 'item_id' => $item_id,
8183 - 'message' => 'Item processed successfully'
8184 - ));
8185 -
8186 - } catch (Exception $e) {
8187 - $error_message = $e->getMessage();
8188 -
8189 - // Get current attempt count
8190 - $item = $wpdb->get_row($wpdb->prepare(
8191 - "SELECT attempts, max_attempts FROM $table_name WHERE id = %d",
8192 - $item_id
8193 - ));
8194 -
8195 - // Check if we've exhausted retries
8196 - if ($item && $item->attempts >= $item->max_attempts) {
8197 - // Permanently failed
8198 - $wpdb->update(
8199 - $table_name,
8200 - array(
8201 - 'status' => 'failed',
8202 - 'error_message' => $error_message
8203 - ),
8204 - array('id' => $item_id),
8205 - array('%s', '%s'),
8206 - array('%d')
8207 - );
8208 -
8209 - wp_send_json_error(array(
8210 - 'message' => 'Item failed after maximum attempts: ' . $error_message,
8211 - 'permanent_failure' => true,
8212 - 'item_id' => $item_id
8213 - ));
8214 - } else {
8215 - // Mark for retry
8216 - $wpdb->update(
8217 - $table_name,
8218 - array(
8219 - 'status' => 'failed',
8220 - 'error_message' => $error_message
8221 - ),
8222 - array('id' => $item_id),
8223 - array('%s', '%s'),
8224 - array('%d')
8225 - );
8226 -
8227 - wp_send_json_error(array(
8228 - 'message' => 'Item processing failed, will retry: ' . $error_message,
8229 - 'can_retry' => true,
8230 - 'item_id' => $item_id,
8231 - 'attempts' => $item ? $item->attempts : 0
8232 - ));
8233 - }
8234 - }
8235 -}
8236 -
8237 -/**
8238 - * Process a URL from the queue
8239 - */
8240 -private function mxchat_process_queue_url($item_data, $bot_id = 'default', $queue_id = '') {
8241 - $url = isset($item_data['url']) ? $item_data['url'] : '';
8242 -
8243 - if (empty($url)) {
8244 - return new WP_Error('invalid_url', 'URL is empty');
8245 - }
8246 -
8247 - // Get bot-specific embedding decision early (needed for both paths) —
8248 - // custom-provider-aware (plan cbd5fd). Error code preserved.
8249 - $bot_options = $this->get_bot_options($bot_id);
8250 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
8251 -
8252 - $preflight = MxChat_Utils::embedding_preflight($options);
8253 - if (!$preflight['ok']) {
8254 - return new WP_Error('no_api_key', $preflight['reason'] . ' (bot: ' . $bot_id . ')');
8255 - }
8256 - $api_key = $preflight['api_key'];
8257 -
8258 - // Check if this is a WooCommerce product URL and WooCommerce is active
8259 - $is_product_url = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
8260 - $content_type = $is_product_url ? 'product' : 'url';
8261 -
8262 - // Try to get WooCommerce product data if it's a product URL
8263 - if ($is_product_url && class_exists('WooCommerce')) {
8264 - $product_content = $this->mxchat_extract_woocommerce_product_content($url);
8265 -
8266 - if (!empty($product_content)) {
8267 - // Successfully extracted WooCommerce product data with pricing
8268 - $result = MxChat_Utils::submit_content_to_db(
8269 - $product_content,
8270 - $url,
8271 - $api_key,
8272 - null,
8273 - $bot_id,
8274 - 'product'
8275 - );
8276 - return $result;
8277 - }
8278 - // If WooCommerce extraction failed, fall through to HTML extraction
8279 - }
8280 -
8281 - // Fetch URL content (fallback for non-products or when WooCommerce extraction fails)
8282 - $is_likely_pdf = (strtolower(pathinfo(parse_url($url, PHP_URL_PATH) ?: '', PATHINFO_EXTENSION)) === 'pdf');
8283 - $response = wp_remote_get($url, array(
8284 - 'timeout' => $is_likely_pdf ? 120 : 30,
8285 - 'redirection' => 5,
8286 - 'user-agent' => mxchat_ingest_user_agent(),
8287 - ));
8288 -
8289 - if (is_wp_error($response)) {
8290 - return $response;
8291 - }
8292 -
8293 - $response_code = wp_remote_retrieve_response_code($response);
8294 - if ($response_code !== 200) {
8295 - return new WP_Error('http_error', 'HTTP ' . $response_code . ' error for: ' . $url);
8296 - }
8297 -
8298 - // Check if URL is a PDF — expand into per-page queue items using the standard PDF pipeline
8299 - if ($this->mxchat_is_pdf_url($url, $response)) {
8300 - return $this->mxchat_expand_pdf_to_queue($url, $response, $bot_id, $queue_id);
8301 - }
8302 -
8303 - $html = wp_remote_retrieve_body($response);
8304 -
8305 - if (empty($html)) {
8306 - return new WP_Error('empty_response', 'Empty response body');
8307 - }
8308 -
8309 - // Extract and sanitize content
8310 - $content = $this->mxchat_extract_main_content($html);
8311 - $sanitized = $this->mxchat_sanitize_content_for_api($content);
8312 -
8313 - if (empty($sanitized)) {
8314 - // Not an error - just no content found (maybe a redirect or empty page)
8315 - return false;
8316 - }
8317 -
8318 - // Submit to database with content_type
8319 - $result = MxChat_Utils::submit_content_to_db(
8320 - $sanitized,
8321 - $url,
8322 - $api_key,
8323 - null,
8324 - $bot_id,
8325 - $content_type
8326 - );
8327 -
8328 - return $result;
8329 -}
8330 -
8331 -/**
8332 - * Expand a PDF URL into per-page queue items using the standard PDF pipeline.
8333 - * Called when a sitemap URL turns out to be a PDF — downloads, parses page count,
8334 - * and adds pdf_page items to the same queue so they process with full progress tracking.
8335 - */
8336 -private function mxchat_expand_pdf_to_queue($pdf_url, $response, $bot_id = 'default', $queue_id = '') {
8337 - set_time_limit(120); // PDFs need extra time for download + parsing
8338 -
8339 - $upload_dir = wp_upload_dir();
8340 - $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
8341 - $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
8342 -
8343 - $response_body = wp_remote_retrieve_body($response);
8344 - if (empty($response_body)) {
8345 - return new WP_Error('empty_pdf', 'Empty PDF response for: ' . $pdf_url);
8346 - }
8347 -
8348 - if (!wp_mkdir_p(dirname($pdf_path))) {
8349 - return new WP_Error('dir_error', 'Failed to create upload directory');
8350 - }
8351 -
8352 - file_put_contents($pdf_path, $response_body);
8353 -
8354 - if (!file_exists($pdf_path)) {
8355 - return new WP_Error('save_error', 'Failed to save PDF file');
8356 - }
8357 -
8358 - try {
8359 - $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
8360 -
8361 - if ($total_pages === false || $total_pages < 1) {
8362 - wp_delete_file($pdf_path);
8363 - return new WP_Error('no_pages', 'PDF has no pages: ' . $pdf_url);
8364 - }
8365 -
8366 - // Build per-page items identical to mxchat_handle_pdf_for_knowledge_base
8367 - $pages = array();
8368 - for ($i = 1; $i <= $total_pages; $i++) {
8369 - $pages[] = array(
8370 - 'pdf_path' => $pdf_path,
8371 - 'pdf_url' => $pdf_url,
8372 - 'page_number' => $i,
8373 - 'total_pages' => $total_pages
8374 - );
8375 - }
8376 -
8377 - // Add pdf_page items to the SAME queue so the JS picks them up automatically
8378 - if (!empty($queue_id)) {
8379 - $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
8380 - } else {
8381 - // Fallback: create a new PDF queue (shouldn't happen in sitemap flow)
8382 - $new_queue_id = 'pdf_' . md5($pdf_url . time());
8383 - $queued_count = $this->mxchat_add_to_queue($new_queue_id, 'pdf_page', $pages, $bot_id);
8384 - $this->mxchat_set_queue_meta($new_queue_id, 'source_url', $pdf_url);
8385 - $this->mxchat_set_queue_meta($new_queue_id, 'queue_type', 'pdf');
8386 - $this->mxchat_set_queue_meta($new_queue_id, 'total_items', $total_pages);
8387 - $this->mxchat_set_queue_meta($new_queue_id, 'bot_id', $bot_id);
8388 - $this->mxchat_set_queue_meta($new_queue_id, 'pdf_path', $pdf_path);
8389 - $this->mxchat_set_queue_meta($new_queue_id, 'created_at', current_time('mysql'));
8390 - }
8391 -
8392 - if ($queued_count === 0) {
8393 - wp_delete_file($pdf_path);
8394 - return new WP_Error('queue_error', 'Failed to add PDF pages to queue');
8395 - }
8396 -
8397 - // Return true so the original URL item is marked complete
8398 - // The new pdf_page items will be processed in subsequent batches
8399 - return true;
8400 -
8401 - } catch (Exception $e) {
8402 - if (file_exists($pdf_path)) {
8403 - wp_delete_file($pdf_path);
8404 - }
8405 - return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
8406 - }
8407 -}
8408 -
8409 -/**
8410 - * Legacy: Process a PDF URL inline during sitemap queue processing.
8411 - * @deprecated Use mxchat_expand_pdf_to_queue instead — kept for reference only.
8412 - */
8413 -private function mxchat_process_pdf_url_inline($pdf_url, $response, $api_key, $bot_id = 'default') {
8414 - set_time_limit(120); // PDFs need more time — downloading + parsing all pages
8415 -
8416 - $upload_dir = wp_upload_dir();
8417 - $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
8418 - $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;
8419 -
8420 - $response_body = wp_remote_retrieve_body($response);
8421 - if (empty($response_body)) {
8422 - return new WP_Error('empty_pdf', 'Empty PDF response');
8423 - }
8424 -
8425 - if (!wp_mkdir_p(dirname($pdf_path))) {
8426 - return new WP_Error('dir_error', 'Failed to create upload directory');
8427 - }
8428 -
8429 - file_put_contents($pdf_path, $response_body);
8430 -
8431 - if (!file_exists($pdf_path)) {
8432 - return new WP_Error('save_error', 'Failed to save PDF file');
8433 - }
8434 -
8435 - try {
8436 - mxchat_load_pdf_parser();
8437 - $parser = new \Smalot\PdfParser\Parser();
8438 - $pdf = $parser->parseFile($pdf_path);
8439 - $pages = $pdf->getPages();
8440 - $total_pages = count($pages);
8441 -
8442 - if ($total_pages < 1) {
8443 - wp_delete_file($pdf_path);
8444 - return new WP_Error('no_pages', 'PDF has no pages');
8445 - }
8446 -
8447 - $processed = 0;
8448 - $skipped_pages = array();
8449 -
8450 - for ($i = 0; $i < $total_pages; $i++) {
8451 - $page_num = $i + 1;
8452 - $text = $pages[$i]->getText();
8453 - if (empty($text)) {
8454 - $skipped_pages[] = 'Page ' . $page_num . ': No text could be extracted — page may contain only images, links, or non-standard encoding';
8455 - continue;
8456 - }
8457 -
8458 - $sanitized = $this->mxchat_sanitize_content_for_api($text);
8459 - if (empty($sanitized)) {
8460 - $skipped_pages[] = 'Page ' . $page_num . ': Text was extracted but contained only special characters, control codes, or unsupported content';
8461 - continue;
8462 - }
8463 -
8464 - $metadata = array(
8465 - 'document_type' => 'pdf',
8466 - 'total_pages' => $total_pages,
8467 - 'current_page' => $page_num,
8468 - 'source_url' => $pdf_url,
8469 - );
8470 -
8471 - $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
8472 - $page_url = esc_url($pdf_url . '#page=' . $page_num);
8473 -
8474 - MxChat_Utils::submit_content_to_db(
8475 - $content_with_metadata,
8476 - $page_url,
8477 - $api_key,
8478 - null,
8479 - $bot_id,
8480 - 'pdf'
8481 - );
8482 -
8483 - $processed++;
8484 - }
8485 -
8486 - // Clean up the temp PDF file
8487 - wp_delete_file($pdf_path);
8488 -
8489 - if (!empty($skipped_pages)) {
8490 - error_log('MxChat PDF: Skipped ' . count($skipped_pages) . ' of ' . $total_pages . ' pages: ' . implode('; ', $skipped_pages));
8491 - }
8492 -
8493 - return $processed > 0 ? true : false;
8494 -
8495 - } catch (Exception $e) {
8496 - if (file_exists($pdf_path)) {
8497 - wp_delete_file($pdf_path);
8498 - }
8499 - return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
8500 - }
8501 -}
8502 -
8503 -/**
8504 - * Extract WooCommerce product content including pricing
8505 - *
8506 - * @param string $url The product URL
8507 - * @return string|false Product content with pricing, or false if not found
8508 - */
8509 -private function mxchat_extract_woocommerce_product_content($url) {
8510 - // Try to get product ID from URL
8511 - $product_id = url_to_postid($url);
8512 -
8513 - // If url_to_postid fails, try to extract from URL pattern
8514 - if (!$product_id) {
8515 - $product_slug = '';
8516 -
8517 - // Handle pretty permalinks: /product/product-name/
8518 - if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
8519 - $product_slug = $matches[1];
8520 - }
8521 -
8522 - if (!empty($product_slug)) {
8523 - $product_post = get_page_by_path($product_slug, OBJECT, 'product');
8524 - if ($product_post) {
8525 - $product_id = $product_post->ID;
8526 - }
8527 - }
8528 - }
8529 -
8530 - if (!$product_id) {
8531 - return false;
8532 - }
8533 -
8534 - // Get WooCommerce product object
8535 - $product = wc_get_product($product_id);
8536 -
8537 - if (!$product) {
8538 - return false;
8539 - }
8540 -
8541 - // Build product content with pricing (similar to mxchat_store_product_embedding)
8542 - $title = $product->get_name();
8543 - $description = $product->get_description();
8544 - $short_description = $product->get_short_description();
8545 - $sku = $product->get_sku();
8546 -
8547 - // Get pricing information
8548 - $regular_price = $product->get_regular_price();
8549 - $sale_price = $product->get_sale_price();
8550 - $price = $product->get_price(); // Current active price
8551 -
8552 - // Get currency symbol
8553 - $currency_symbol = get_woocommerce_currency_symbol();
8554 -
8555 - // Format content
8556 - $content = $title . "\n\n";
8557 -
8558 - if (!empty($short_description)) {
8559 - $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
8560 - }
8561 -
8562 - if (!empty($description)) {
8563 - $content .= wp_strip_all_tags($description) . "\n\n";
8564 - }
8565 -
8566 - // Add pricing information
8567 - if (!empty($regular_price)) {
8568 - $content .= "Price: " . $currency_symbol . $regular_price . "\n";
8569 - } elseif (!empty($price)) {
8570 - $content .= "Price: " . $currency_symbol . $price . "\n";
8571 - }
8572 -
8573 - if (!empty($sale_price) && $sale_price !== $regular_price) {
8574 - $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n";
8575 - }
8576 -
8577 - // Handle variable products - show price range
8578 - if ($product->is_type('variable')) {
8579 - $min_price = $product->get_variation_price('min');
8580 - $max_price = $product->get_variation_price('max');
8581 - if ($min_price !== $max_price) {
8582 - $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n";
8583 - }
8584 - }
8585 -
8586 - if (!empty($sku)) {
8587 - $content .= "SKU: " . $sku . "\n";
8588 - }
8589 -
8590 - // Get product categories
8591 - $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names'));
8592 - if (!empty($categories) && !is_wp_error($categories)) {
8593 - $content .= "Categories: " . implode(', ', $categories) . "\n";
8594 - }
8595 -
8596 - // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots)
8597 - $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
8598 - if (!empty($custom_tabs) && is_array($custom_tabs)) {
8599 - foreach ($custom_tabs as $tab) {
8600 - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
8601 - $tab_content = isset($tab['content']) ? $tab['content'] : '';
8602 -
8603 - if (!empty($tab_title) && !empty($tab_content)) {
8604 - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
8605 - }
8606 - }
8607 - }
8608 -
8609 - // Also check for reusable/saved tabs applied to this product
8610 - $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
8611 - if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
8612 - $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
8613 - if (!empty($saved_tabs) && is_array($saved_tabs)) {
8614 - foreach ($applied_saved_tabs as $saved_tab_id) {
8615 - if (isset($saved_tabs[$saved_tab_id])) {
8616 - $tab = $saved_tabs[$saved_tab_id];
8617 - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
8618 - $tab_content = isset($tab['content']) ? $tab['content'] : '';
8619 -
8620 - if (!empty($tab_title) && !empty($tab_content)) {
8621 - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
8622 - }
8623 - }
8624 - }
8625 - }
8626 - }
8627 -
8628 - return $this->mxchat_sanitize_content_for_api($content);
8629 -}
8630 -
8631 -/**
8632 - * Process a PDF page from the queue
8633 - */
8634 -private function mxchat_process_queue_pdf_page($item_data, $bot_id = 'default') {
8635 - $pdf_path = isset($item_data['pdf_path']) ? $item_data['pdf_path'] : '';
8636 - $pdf_url = isset($item_data['pdf_url']) ? $item_data['pdf_url'] : '';
8637 - $page_number = isset($item_data['page_number']) ? absint($item_data['page_number']) : 0;
8638 - $total_pages = isset($item_data['total_pages']) ? absint($item_data['total_pages']) : 0;
8639 -
8640 - if (empty($pdf_path) || !file_exists($pdf_path)) {
8641 - return new WP_Error('pdf_not_found', 'PDF file not found: ' . $pdf_path);
8642 - }
8643 -
8644 - if ($page_number < 1) {
8645 - return new WP_Error('invalid_page', 'Invalid page number');
8646 - }
8647 -
8648 - try {
8649 - mxchat_load_pdf_parser();
8650 - $parser = new \Smalot\PdfParser\Parser();
8651 - $pdf = $parser->parseFile($pdf_path);
8652 - $pages = $pdf->getPages();
8653 -
8654 - if (!isset($pages[$page_number - 1])) {
8655 - return new WP_Error('page_not_found', 'Page ' . $page_number . ' not found in PDF');
8656 - }
8657 -
8658 - $text = $pages[$page_number - 1]->getText();
8659 -
8660 - if (empty($text)) {
8661 - return new WP_Error('empty_page', 'Page ' . $page_number . ': No text could be extracted — page may contain only images, links, or non-standard encoding');
8662 - }
8663 -
8664 - $sanitized = $this->mxchat_sanitize_content_for_api($text);
8665 -
8666 - if (empty($sanitized)) {
8667 - return new WP_Error('empty_after_sanitization', 'Page ' . $page_number . ': Text was extracted but contained only special characters, control codes, or unsupported content that was removed during cleanup');
8668 - }
8669 -
8670 - // Create metadata
8671 - $metadata = array(
8672 - 'document_type' => 'pdf',
8673 - 'total_pages' => $total_pages,
8674 - 'current_page' => $page_number,
8675 - 'source_url' => $pdf_url
8676 - );
8677 -
8678 - $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
8679 - $page_url = esc_url($pdf_url . "#page=" . $page_number);
8680 -
8681 - // Get bot-specific embedding decision — custom-provider-aware
8682 - // (plan cbd5fd). Error code preserved.
8683 - $bot_options = $this->get_bot_options($bot_id);
8684 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
8685 -
8686 - $preflight = MxChat_Utils::embedding_preflight($options);
8687 - if (!$preflight['ok']) {
8688 - return new WP_Error('no_api_key', $preflight['reason'] . ' (bot: ' . $bot_id . ')');
8689 - }
8690 - $api_key = $preflight['api_key'];
8691 -
8692 - // Submit to database - UPDATED 2.5.6: Added content_type 'pdf'
8693 - $result = MxChat_Utils::submit_content_to_db(
8694 - $content_with_metadata,
8695 - $page_url,
8696 - $api_key,
8697 - null,
8698 - $bot_id,
8699 - 'pdf'
8700 - );
8701 -
8702 - return $result;
8703 -
8704 - } catch (Exception $e) {
8705 - return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
8706 - }
8707 -}
8708 -
8709 -/**
8710 - * AJAX: Get queue processing status
8711 - */
8712 -public function ajax_mxchat_get_queue_status() {
8713 - // Verify nonce and permissions
8714 - check_ajax_referer('mxchat_queue_nonce', 'nonce');
8715 -
8716 - if (!current_user_can('manage_options')) {
8717 - wp_send_json_error('Unauthorized access');
8718 - }
8719 -
8720 - $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8721 -
8722 - if (empty($queue_id)) {
8723 - wp_send_json_error('Missing queue ID');
8724 - }
8725 -
8726 - global $wpdb;
8727 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8728 -
8729 - // Get counts by status
8730 - $counts = $wpdb->get_results($wpdb->prepare(
8731 - "SELECT status, COUNT(*) as count
8732 - FROM $table_name
8733 - WHERE queue_id = %s
8734 - GROUP BY status",
8735 - $queue_id
8736 - ), OBJECT_K);
8737 -
8738 - $total = 0;
8739 - $completed = 0;
8740 - $failed = 0;
8741 - $processing = 0;
8742 - $pending = 0;
8743 -
8744 - foreach ($counts as $status => $data) {
8745 - $count = absint($data->count);
8746 - $total += $count;
8747 -
8748 - switch ($status) {
8749 - case 'completed':
8750 - $completed = $count;
8751 - break;
8752 - case 'failed':
8753 - $failed = $count;
8754 - break;
8755 - case 'processing':
8756 - $processing = $count;
8757 - break;
8758 - case 'pending':
8759 - $pending = $count;
8760 - break;
8761 - }
8762 - }
8763 -
8764 - // Calculate percentage
8765 - $percentage = $total > 0 ? round((($completed + $failed) / $total) * 100) : 0;
8766 -
8767 - // Get failed items details (include all failed items, not just those that exhausted retries)
8768 - $failed_items = array();
8769 - if ($failed > 0) {
8770 - $failed_items = $wpdb->get_results($wpdb->prepare(
8771 - "SELECT item_type, item_data, error_message, attempts
8772 - FROM $table_name
8773 - WHERE queue_id = %s
8774 - AND status = 'failed'
8775 - ORDER BY id DESC
8776 - LIMIT 50",
8777 - $queue_id
8778 - ));
8779 - }
8780 -
8781 - // Get queue metadata
8782 - $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
8783 - $queue_type = $this->mxchat_get_queue_meta($queue_id, 'queue_type');
8784 -
8785 - // Determine if queue is complete
8786 - $is_complete = ($pending === 0 && $processing === 0);
8787 -
8788 - wp_send_json_success(array(
8789 - 'queue_id' => $queue_id,
8790 - 'queue_type' => $queue_type,
8791 - 'source_url' => $source_url,
8792 - 'total' => $total,
8793 - 'completed' => $completed,
8794 - 'failed' => $failed,
8795 - 'processing' => $processing,
8796 - 'pending' => $pending,
8797 - 'percentage' => $percentage,
8798 - 'is_complete' => $is_complete,
8799 - 'failed_items' => $failed_items,
8800 - 'status' => $is_complete ? 'complete' : 'processing'
8801 - ));
8802 -}
8803 -
8804 -/**
8805 - * AJAX: Clear completed queue
8806 - */
8807 -public function ajax_mxchat_clear_queue() {
8808 - // Verify nonce and permissions
8809 - check_ajax_referer('mxchat_queue_nonce', 'nonce');
8810 -
8811 - if (!current_user_can('manage_options')) {
8812 - wp_send_json_error('Unauthorized access');
8813 - }
8814 -
8815 - $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8816 -
8817 - if (empty($queue_id)) {
8818 - wp_send_json_error('Missing queue ID');
8819 - }
8820 -
8821 - global $wpdb;
8822 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8823 - $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
8824 -
8825 - // Delete queue items
8826 - $wpdb->delete(
8827 - $table_name,
8828 - array('queue_id' => $queue_id),
8829 - array('%s')
8830 - );
8831 -
8832 - // Delete queue metadata
8833 - $wpdb->delete(
8834 - $meta_table,
8835 - array('queue_id' => $queue_id),
8836 - array('%s')
8837 - );
8838 -
8839 - wp_send_json_success(array(
8840 - 'message' => 'Queue cleared successfully'
8841 - ));
8842 -}
8843 -
8844 -/**
8845 - * AJAX: Retry failed items in queue
8846 - */
8847 -public function ajax_mxchat_retry_failed() {
8848 - // Verify nonce and permissions
8849 - check_ajax_referer('mxchat_queue_nonce', 'nonce');
8850 -
8851 - if (!current_user_can('manage_options')) {
8852 - wp_send_json_error('Unauthorized access');
8853 - }
8854 -
8855 - $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8856 -
8857 - if (empty($queue_id)) {
8858 - wp_send_json_error('Missing queue ID');
8859 - }
8860 -
8861 - global $wpdb;
8862 - $table_name = $wpdb->prefix . 'mxchat_processing_queue';
8863 -
8864 - // Reset failed items to pending and reset attempt count
8865 - $updated = $wpdb->update(
8866 - $table_name,
8867 - array(
8868 - 'status' => 'pending',
8869 - 'attempts' => 0,
8870 - 'error_message' => null
8871 - ),
8872 - array(
8873 - 'queue_id' => $queue_id,
8874 - 'status' => 'failed'
8875 - ),
8876 - array('%s', '%d', '%s'),
8877 - array('%s', '%s')
8878 - );
8879 -
8880 - wp_send_json_success(array(
8881 - 'message' => 'Reset ' . $updated . ' failed items for retry',
8882 - 'reset_count' => $updated
8883 - ));
8884 -}
8885 -
8886 -
8887 -public function ajax_mxchat_mark_queue_complete() {
8888 - check_ajax_referer('mxchat_queue_nonce', 'nonce');
8889 -
8890 - if (!current_user_can('manage_options')) {
8891 - wp_send_json_error('Unauthorized access');
8892 - }
8893 -
8894 - $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
8895 -
8896 - if (empty($queue_id)) {
8897 - wp_send_json_error('Missing queue ID');
8898 - }
8899 -
8900 - // Clear active queue transients
8901 - if (strpos($queue_id, 'sitemap_') === 0) {
8902 - delete_transient('mxchat_active_queue_sitemap');
8903 - } else if (strpos($queue_id, 'pdf_') === 0) {
8904 - delete_transient('mxchat_active_queue_pdf');
8905 - }
8906 -
8907 - wp_send_json_success(array('message' => 'Queue marked as complete'));
8908 -}
8909 -
8910 3553
8911 3554 // ========================================
8912 3555 // STATIC ACCESS METHODS
8913 3556 // ========================================