| @@ -9,11 +9,27 @@ | ||
| 9 | 9 | exit; // Exit if accessed directly |
| 10 | 10 | } |
| 11 | 11 | |
| 12 | 12 | class MxChat_Knowledge_Manager { |
| 13 | - | |
| 13 | + | |
| 14 | 14 | private $options; |
| 15 | - | |
| 15 | + | |
| 16 | + // Post IDs whose vectors were already deleted by mxchat_handle_status_transition this | |
| 17 | + // request, so the transient-based branch in mxchat_handle_post_update can skip the | |
| 18 | + // redundant (idempotent but network-visible) second deletion. | |
| 19 | + private $transition_deleted_posts = array(); | |
| 20 | + | |
| 21 | + // Post IDs already INDEXED by mxchat_handle_status_transition's arrival edge this | |
| 22 | + // request. Normal editor publishes fire transition_post_status first, then | |
| 23 | + // post_updated — without this guard every editor publish would embed twice. | |
| 24 | + private $transition_indexed_posts = array(); | |
| 25 | + | |
| 26 | + // Post IDs core has announced an in-flight UPDATE for. pre_post_update fires only | |
| 27 | + // inside wp_insert_post's update branch and always before wp_transition_post_status, | |
| 28 | + // so this is an exact "a post_updated is coming later this request" signal — which is | |
| 29 | + // what makes it safe to arm transition_indexed_posts (plan a664f3). | |
| 30 | + private $pending_post_update = array(); | |
| 31 | + | |
| 16 | 32 | /** |
| 17 | 33 | * Constructor - Register hooks for content processing |
| 18 | 34 | */ |
| 19 | 35 | public function __construct() { |
| @@ -31,8 +47,10 @@ | ||
| 31 | 47 | // Admin post handlers for form submissions |
| 32 | 48 | add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission')); |
| 33 | 49 | add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission')); |
| 34 | 50 | add_action('admin_post_mxchat_submit_pdf_file', array($this, 'mxchat_handle_pdf_file_submission')); |
| 51 | + add_action('admin_post_mxchat_submit_document_file', array($this, 'mxchat_handle_document_file_submission')); | |
| 52 | + add_action('admin_post_mxchat_submit_youtube', array($this, 'mxchat_handle_youtube_submission')); | |
| 35 | 53 | add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing')); |
| 36 | 54 | |
| 37 | 55 | // AJAX handlers for real-time processing and status updates |
| 38 | 56 | add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates')); |
| @@ -65,13 +83,27 @@ | ||
| 65 | 83 | add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2); |
| 66 | 84 | add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3); |
| 67 | 85 | add_action('before_delete_post', array($this, 'mxchat_handle_post_delete')); |
| 68 | 86 | add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete')); |
| 87 | + // Authoritative unpublish detection: core hands this hook the REAL previous status, so | |
| 88 | + // removal no longer depends on the mxchat_prev_status_* transients (evictable by persistent | |
| 89 | + // object caches, never written by paths that bypass wp_update_post, e.g. plugins flipping | |
| 90 | + // post_status directly and calling wp_transition_post_status themselves). | |
| 91 | + add_action('transition_post_status', array($this, 'mxchat_handle_status_transition'), 10, 3); | |
| 69 | 92 | |
| 70 | 93 | // ACF hook - fires AFTER ACF fields are saved, ensuring ACF data is available |
| 71 | 94 | // Priority 20 to run after ACF's own save (which runs at priority 10) |
| 72 | 95 | add_action('acf/save_post', array($this, 'mxchat_handle_acf_save'), 20); |
| 73 | 96 | |
| 97 | + // One-time cleanup for vectors orphaned by unpublishes that predate the | |
| 98 | + // transition_post_status handler (plan 816fb1): wp mxchat prune-unpublished | |
| 99 | + if (defined('WP_CLI') && WP_CLI) { | |
| 100 | + WP_CLI::add_command('mxchat prune-unpublished', array($this, 'cli_prune_unpublished')); | |
| 101 | + // In-place repair for RTL KB rows imported in visual order before the | |
| 102 | + // 32bf9e normalizer existed: wp mxchat rtl-repair (plan d1e6f7) | |
| 103 | + WP_CLI::add_command('mxchat rtl-repair', array($this, 'cli_rtl_repair')); | |
| 104 | + } | |
| 105 | + | |
| 74 | 106 | add_action('wp_ajax_mxchat_mark_queue_complete', array($this, 'ajax_mxchat_mark_queue_complete')); |
| 75 | 107 | |
| 76 | 108 | // WooCommerce product hooks (if WooCommerce is active) |
| 77 | 109 | if (class_exists('WooCommerce')) { |
| @@ -119,26 +151,17 @@ | ||
| 119 | 151 | |
| 120 | 152 | // Get bot-specific options and API key |
| 121 | 153 | $bot_options = $this->get_bot_options($bot_id); |
| 122 | 154 | $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options'); |
| 123 | - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002'; | |
| 124 | - | |
| 125 | - if (strpos($selected_model, 'voyage') === 0) { | |
| 126 | - $api_key = $options['voyage_api_key'] ?? ''; | |
| 127 | - } elseif (strpos($selected_model, 'gemini-embedding') === 0) { | |
| 128 | - $api_key = $options['gemini_api_key'] ?? ''; | |
| 129 | - } else { | |
| 130 | - $api_key = $options['api_key'] ?? ''; | |
| 131 | - } | |
| 132 | - | |
| 133 | - if (empty($api_key)) { | |
| 134 | - set_transient('mxchat_admin_notice_error', | |
| 135 | - esc_html__('API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'), | |
| 136 | - 30 | |
| 137 | - ); | |
| 155 | + | |
| 156 | + // Custom-provider-aware decision; keyless custom sites must pass (plan cbd5fd). | |
| 157 | + $preflight = MxChat_Utils::embedding_preflight($options); | |
| 158 | + if (!$preflight['ok']) { | |
| 159 | + set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30); | |
| 138 | 160 | wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts'))); |
| 139 | 161 | exit; |
| 140 | 162 | } |
| 163 | + $api_key = $preflight['api_key']; | |
| 141 | 164 | |
| 142 | 165 | // Use centralized utility function with bot_id |
| 143 | 166 | $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key, null, $bot_id); |
| 144 | 167 | |
| @@ -157,8 +180,238 @@ | ||
| 157 | 180 | wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts'))); |
| 158 | 181 | exit; |
| 159 | 182 | } |
| 160 | 183 | |
| 184 | +/** | |
| 185 | + * Handle the "YouTube" KB import source (admin-post form submission). | |
| 186 | + * | |
| 187 | + * Per-video description mode: | |
| 188 | + * - auto: fetch oEmbed metadata (reliable) + best-effort captions transcript. | |
| 189 | + * If no usable transcript, index the metadata anyway, tell the admin, | |
| 190 | + * and bounce back with the manual box pre-filled (never fail silently). | |
| 191 | + * - manual: the admin's own description is what gets indexed; metadata rides along. | |
| 192 | + * | |
| 193 | + * The row is stored with content_type 'youtube' and source_url = the canonical | |
| 194 | + * watch URL, so re-importing the same video UPDATES the entry (source_url | |
| 195 | + * duplicate handling in MxChat_Utils::store_in_wordpress_db) — that is also the | |
| 196 | + * "augment a metadata-only entry" path. | |
| 197 | + */ | |
| 198 | +public function mxchat_handle_youtube_submission() { | |
| 199 | + if (!isset($_POST['submit_youtube']) || !current_user_can('manage_options')) { | |
| 200 | + wp_die(esc_html__('Unauthorized access', 'mxchat')); | |
| 201 | + } | |
| 202 | + | |
| 203 | + check_admin_referer('mxchat_submit_youtube_action', 'mxchat_submit_youtube_nonce'); | |
| 204 | + | |
| 205 | + $redirect_url = admin_url('admin.php?page=mxchat-prompts'); | |
| 206 | + | |
| 207 | + $youtube_url = isset($_POST['youtube_url']) ? esc_url_raw(wp_unslash($_POST['youtube_url'])) : ''; | |
| 208 | + $video_id = MxChat_Utils::parse_youtube_id($youtube_url); | |
| 209 | + | |
| 210 | + if (empty($video_id)) { | |
| 211 | + set_transient('mxchat_admin_notice_error', | |
| 212 | + esc_html__('That does not look like a link to a single YouTube video. Please paste a watch, youtu.be, or Shorts URL.', 'mxchat'), | |
| 213 | + 30 | |
| 214 | + ); | |
| 215 | + wp_safe_redirect(esc_url($redirect_url)); | |
| 216 | + exit; | |
| 217 | + } | |
| 218 | + | |
| 219 | + $canonical_url = 'https://www.youtube.com/watch?v=' . $video_id; | |
| 220 | + | |
| 221 | + $description_mode = (isset($_POST['youtube_description_mode']) && $_POST['youtube_description_mode'] === 'manual') ? 'manual' : 'auto'; | |
| 222 | + $manual_description = isset($_POST['youtube_description']) ? trim(wp_kses_post(wp_unslash($_POST['youtube_description']))) : ''; | |
| 223 | + | |
| 224 | + $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default'; | |
| 225 | + | |
| 226 | + // Resolve the embedding decision exactly like the sibling handlers — | |
| 227 | + // custom-provider-aware (plan cbd5fd). | |
| 228 | + $bot_options = $this->get_bot_options($bot_id); | |
| 229 | + $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options'); | |
| 230 | + | |
| 231 | + $preflight = MxChat_Utils::embedding_preflight($options); | |
| 232 | + if (!$preflight['ok']) { | |
| 233 | + set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30); | |
| 234 | + wp_safe_redirect(esc_url($redirect_url)); | |
| 235 | + exit; | |
| 236 | + } | |
| 237 | + $api_key = $preflight['api_key']; | |
| 238 | + | |
| 239 | + // Metadata is fetched in BOTH modes — it is the reliable half of auto, and in | |
| 240 | + // manual mode it enriches the indexed text with the real title/channel. | |
| 241 | + $meta = $this->mxchat_fetch_youtube_oembed($video_id); | |
| 242 | + $video_title = isset($meta['title']) ? sanitize_text_field($meta['title']) : ''; | |
| 243 | + $video_channel = isset($meta['author_name']) ? sanitize_text_field($meta['author_name']) : ''; | |
| 244 | + | |
| 245 | + $header_lines = 'YouTube Video: ' . ($video_title !== '' ? $video_title : $canonical_url) . "\n"; | |
| 246 | + if ($video_channel !== '') { | |
| 247 | + $header_lines .= 'Channel: ' . $video_channel . "\n"; | |
| 248 | + } | |
| 249 | + $header_lines .= 'URL: ' . $canonical_url . "\n\n"; | |
| 250 | + | |
| 251 | + $transcript_missing = false; | |
| 252 | + | |
| 253 | + if ($description_mode === 'manual') { | |
| 254 | + if ($manual_description === '') { | |
| 255 | + set_transient('mxchat_admin_notice_error', | |
| 256 | + esc_html__('Please write a description for the video, or switch to auto-fetch.', 'mxchat'), | |
| 257 | + 30 | |
| 258 | + ); | |
| 259 | + wp_safe_redirect(esc_url($redirect_url)); | |
| 260 | + exit; | |
| 261 | + } | |
| 262 | + $indexed_text = $header_lines . $manual_description; | |
| 263 | + } else { | |
| 264 | + $transcript = $this->mxchat_fetch_youtube_transcript($video_id); | |
| 265 | + | |
| 266 | + if (strlen($transcript) >= 200) { | |
| 267 | + $indexed_text = $header_lines . $transcript; | |
| 268 | + } else { | |
| 269 | + // Graceful fallback: captions disabled / blocked / no speech. Auto | |
| 270 | + // reliably gets metadata; it does NOT guarantee a transcript. | |
| 271 | + $transcript_missing = true; | |
| 272 | + | |
| 273 | + if ($video_title === '' && $video_channel === '') { | |
| 274 | + // Both halves failed — nothing meaningful to index. | |
| 275 | + set_transient('mxchat_admin_notice_error', | |
| 276 | + esc_html__('Could not retrieve any information for that video (no metadata and no captions). Please check the URL, or use the manual description option.', 'mxchat'), | |
| 277 | + 30 | |
| 278 | + ); | |
| 279 | + wp_safe_redirect(esc_url($redirect_url)); | |
| 280 | + exit; | |
| 281 | + } | |
| 282 | + | |
| 283 | + $indexed_text = $header_lines . sprintf( | |
| 284 | + /* translators: 1: video title, 2: channel name */ | |
| 285 | + __('A YouTube video titled "%1$s" from the channel %2$s.', 'mxchat'), | |
| 286 | + $video_title !== '' ? $video_title : $canonical_url, | |
| 287 | + $video_channel !== '' ? $video_channel : 'YouTube' | |
| 288 | + ); | |
| 289 | + } | |
| 290 | + } | |
| 291 | + | |
| 292 | + $result = MxChat_Utils::submit_content_to_db($indexed_text, $canonical_url, $api_key, null, $bot_id, 'youtube'); | |
| 293 | + | |
| 294 | + if (is_wp_error($result)) { | |
| 295 | + set_transient('mxchat_admin_notice_error', | |
| 296 | + esc_html__('Error storing video in the knowledge base: ', 'mxchat') . $result->get_error_message(), | |
| 297 | + 30 | |
| 298 | + ); | |
| 299 | + wp_safe_redirect(esc_url($redirect_url)); | |
| 300 | + exit; | |
| 301 | + } | |
| 302 | + | |
| 303 | + if ($transcript_missing) { | |
| 304 | + set_transient('mxchat_admin_notice_success', | |
| 305 | + esc_html__('Video indexed from its title and channel — no captions were available for a transcript. The form below is pre-filled: write your own description and import again to improve matching (it updates the same entry).', 'mxchat'), | |
| 306 | + 30 | |
| 307 | + ); | |
| 308 | + // Bounce back with prefill args so the page reopens the YouTube form in | |
| 309 | + // manual mode with the URL + fetched title ready to augment. | |
| 310 | + $redirect_url = add_query_arg(array( | |
| 311 | + 'mxchat_yt_prefill' => '1', | |
| 312 | + 'yt_url' => rawurlencode($canonical_url), | |
| 313 | + 'yt_title' => rawurlencode($video_title), | |
| 314 | + ), $redirect_url); | |
| 315 | + } else { | |
| 316 | + set_transient('mxchat_admin_notice_success', | |
| 317 | + esc_html__('YouTube video successfully added to the knowledge base!', 'mxchat'), | |
| 318 | + 30 | |
| 319 | + ); | |
| 320 | + } | |
| 321 | + | |
| 322 | + wp_safe_redirect(esc_url_raw($redirect_url)); | |
| 323 | + exit; | |
| 324 | +} | |
| 325 | + | |
| 326 | +/** | |
| 327 | + * Fetch YouTube oEmbed metadata for a video (no API key required). | |
| 328 | + * Returns the decoded array (title, author_name, thumbnail_url, ...) or array(). | |
| 329 | + */ | |
| 330 | +private function mxchat_fetch_youtube_oembed($video_id) { | |
| 331 | + $oembed_url = 'https://www.youtube.com/oembed?url=' . rawurlencode('https://www.youtube.com/watch?v=' . $video_id) . '&format=json'; | |
| 332 | + $response = wp_remote_get($oembed_url, array('timeout' => 15)); | |
| 333 | + if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) { | |
| 334 | + return array(); | |
| 335 | + } | |
| 336 | + $data = json_decode(wp_remote_retrieve_body($response), true); | |
| 337 | + return is_array($data) ? $data : array(); | |
| 338 | +} | |
| 339 | + | |
| 340 | +/** | |
| 341 | + * Best-effort captions transcript for a video. Deliberately ISOLATED: this uses | |
| 342 | + * YouTube's unofficial timedtext route (the caption track list embedded in the | |
| 343 | + * watch page), which YouTube has broken before and will break again. Every | |
| 344 | + * failure mode returns '' so a break degrades to the metadata-only import path | |
| 345 | + * instead of erroring the whole submission. Do not let anything in here throw. | |
| 346 | + */ | |
| 347 | +private function mxchat_fetch_youtube_transcript($video_id) { | |
| 348 | + $watch_url = 'https://www.youtube.com/watch?v=' . $video_id . '&hl=en'; | |
| 349 | + | |
| 350 | + // First try the honest ingest UA; some responses omit the player config for | |
| 351 | + // bot UAs, so retry once with a browser UA before giving up. | |
| 352 | + $user_agents = array( | |
| 353 | + mxchat_ingest_user_agent(), | |
| 354 | + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', | |
| 355 | + ); | |
| 356 | + | |
| 357 | + $tracks = array(); | |
| 358 | + foreach ($user_agents as $ua) { | |
| 359 | + $response = wp_remote_get($watch_url, array( | |
| 360 | + 'timeout' => 20, | |
| 361 | + 'user-agent' => $ua, | |
| 362 | + )); | |
| 363 | + if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) { | |
| 364 | + continue; | |
| 365 | + } | |
| 366 | + $body = wp_remote_retrieve_body($response); | |
| 367 | + if (!is_string($body) || $body === '' || !preg_match('/"captionTracks":(\[.*?\])(?=,")/s', $body, $m)) { | |
| 368 | + continue; | |
| 369 | + } | |
| 370 | + $decoded = json_decode($m[1], true); | |
| 371 | + if (is_array($decoded) && !empty($decoded)) { | |
| 372 | + $tracks = $decoded; | |
| 373 | + break; | |
| 374 | + } | |
| 375 | + } | |
| 376 | + | |
| 377 | + if (empty($tracks)) { | |
| 378 | + return ''; | |
| 379 | + } | |
| 380 | + | |
| 381 | + // Prefer an English track, else take the first offered. | |
| 382 | + $chosen = null; | |
| 383 | + foreach ($tracks as $track) { | |
| 384 | + if (isset($track['languageCode']) && strpos($track['languageCode'], 'en') === 0) { | |
| 385 | + $chosen = $track; | |
| 386 | + break; | |
| 387 | + } | |
| 388 | + } | |
| 389 | + if ($chosen === null) { | |
| 390 | + $chosen = $tracks[0]; | |
| 391 | + } | |
| 392 | + if (empty($chosen['baseUrl']) || !is_string($chosen['baseUrl'])) { | |
| 393 | + return ''; | |
| 394 | + } | |
| 395 | + | |
| 396 | + $timedtext = wp_remote_get($chosen['baseUrl'], array('timeout' => 20)); | |
| 397 | + if (is_wp_error($timedtext) || wp_remote_retrieve_response_code($timedtext) !== 200) { | |
| 398 | + return ''; | |
| 399 | + } | |
| 400 | + $xml = wp_remote_retrieve_body($timedtext); | |
| 401 | + if (!is_string($xml) || strpos($xml, '<text') === false) { | |
| 402 | + return ''; | |
| 403 | + } | |
| 404 | + | |
| 405 | + // <text start=".." dur="..">caption</text> — strip tags, decode the | |
| 406 | + // double-encoded entities timedtext ships, collapse whitespace. | |
| 407 | + $text = preg_replace('/<[^>]+>/', ' ', $xml); | |
| 408 | + $text = html_entity_decode(html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8'), ENT_QUOTES | ENT_HTML5, 'UTF-8'); | |
| 409 | + $text = trim(preg_replace('/\s+/u', ' ', $text)); | |
| 410 | + | |
| 411 | + return $text; | |
| 412 | +} | |
| 413 | + | |
| 161 | 414 | public function mxchat_is_pdf_url($url, $response) { |
| 162 | 415 | $content_type = wp_remote_retrieve_header($response, 'content-type'); |
| 163 | 416 | $file_extension = strtolower(pathinfo($url, PATHINFO_EXTENSION)); |
| 164 | 417 | |
| @@ -404,8 +657,160 @@ | ||
| 404 | 657 | exit; |
| 405 | 658 | } |
| 406 | 659 | |
| 407 | 660 | /** |
| 661 | + * Handle direct document upload (.docx / .txt / .md) from the knowledge base | |
| 662 | + * page (plan 0485e5). Unlike PDF Upload there is no per-page queue: the text | |
| 663 | + * extracts in one pass and routes through submit_content_to_db, whose chunker | |
| 664 | + * takes over for long content. The uploaded file is read from the PHP temp | |
| 665 | + * file and never persisted — only its extracted text enters the KB. | |
| 666 | + * | |
| 667 | + * Source identity matches PDF Upload's scheme: upload://<filename>, stable | |
| 668 | + * across re-uploads so a re-import REPLACES (delete_chunks_for_url + upsert | |
| 669 | + * per identity) instead of duplicating. | |
| 670 | + */ | |
| 671 | +public function mxchat_handle_document_file_submission() { | |
| 672 | + if (!isset($_POST['submit_document_file']) || !current_user_can('manage_options')) { | |
| 673 | + wp_die(esc_html__('Unauthorized access', 'mxchat')); | |
| 674 | + } | |
| 675 | + | |
| 676 | + check_admin_referer('mxchat_submit_document_file_action', 'mxchat_submit_document_file_nonce'); | |
| 677 | + | |
| 678 | + $redirect_url = admin_url('admin.php?page=mxchat-prompts'); | |
| 679 | + | |
| 680 | + if (empty($_FILES['document_file']) || $_FILES['document_file']['error'] !== UPLOAD_ERR_OK) { | |
| 681 | + $error_code = isset($_FILES['document_file']['error']) ? $_FILES['document_file']['error'] : UPLOAD_ERR_NO_FILE; | |
| 682 | + $error_messages = array( | |
| 683 | + UPLOAD_ERR_INI_SIZE => __('The uploaded file exceeds the server upload_max_filesize limit.', 'mxchat'), | |
| 684 | + UPLOAD_ERR_FORM_SIZE => __('The uploaded file exceeds the form MAX_FILE_SIZE limit.', 'mxchat'), | |
| 685 | + UPLOAD_ERR_PARTIAL => __('The file was only partially uploaded.', 'mxchat'), | |
| 686 | + UPLOAD_ERR_NO_FILE => __('No file was uploaded. Please select a document.', 'mxchat'), | |
| 687 | + UPLOAD_ERR_NO_TMP_DIR => __('Server missing temporary folder.', 'mxchat'), | |
| 688 | + UPLOAD_ERR_CANT_WRITE => __('Server failed to write file to disk.', 'mxchat'), | |
| 689 | + ); | |
| 690 | + $error_msg = isset($error_messages[$error_code]) ? $error_messages[$error_code] : __('Unknown upload error.', 'mxchat'); | |
| 691 | + set_transient('mxchat_admin_notice_error', $error_msg, 30); | |
| 692 | + wp_safe_redirect(esc_url($redirect_url)); | |
| 693 | + exit; | |
| 694 | + } | |
| 695 | + | |
| 696 | + $file = $_FILES['document_file']; | |
| 697 | + $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION)); | |
| 698 | + | |
| 699 | + $finfo = finfo_open(FILEINFO_MIME_TYPE); | |
| 700 | + $mime_type = finfo_file($finfo, $file['tmp_name']); | |
| 701 | + finfo_close($finfo); | |
| 702 | + | |
| 703 | + // Per-extension MIME expectations. finfo commonly reports .docx as | |
| 704 | + // application/zip (it IS a Zip container) and .md as plain text. | |
| 705 | + $mime_ok = false; | |
| 706 | + if ($ext === 'docx') { | |
| 707 | + $mime_ok = in_array($mime_type, array( | |
| 708 | + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', | |
| 709 | + 'application/zip', | |
| 710 | + ), true); | |
| 711 | + } elseif ($ext === 'txt' || $ext === 'md') { | |
| 712 | + $mime_ok = (strpos((string) $mime_type, 'text/') === 0); | |
| 713 | + } | |
| 714 | + | |
| 715 | + if (!$mime_ok) { | |
| 716 | + set_transient('mxchat_admin_notice_error', | |
| 717 | + esc_html__('Invalid or unreadable document. Accepted types: .docx, .txt, .md.', 'mxchat'), | |
| 718 | + 30 | |
| 719 | + ); | |
| 720 | + wp_safe_redirect(esc_url($redirect_url)); | |
| 721 | + exit; | |
| 722 | + } | |
| 723 | + | |
| 724 | + $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default'; | |
| 725 | + $original_filename = sanitize_file_name($file['name']); | |
| 726 | + | |
| 727 | + // ---- Extract text (ONE extractor for .docx — the word handler's) ---- | |
| 728 | + if ($ext === 'docx') { | |
| 729 | + $text = MXChat_Word_Handler::extract_docx_text($file['tmp_name']); | |
| 730 | + if ($text === false) { | |
| 731 | + set_transient('mxchat_admin_notice_error', | |
| 732 | + esc_html__('The .docx file could not be read. It may be corrupt, empty, or not a real Word document.', 'mxchat'), | |
| 733 | + 30 | |
| 734 | + ); | |
| 735 | + wp_safe_redirect(esc_url($redirect_url)); | |
| 736 | + exit; | |
| 737 | + } | |
| 738 | + } else { | |
| 739 | + // .txt / .md read as-is. Markdown keeps its syntax on purpose — | |
| 740 | + // headings are useful retrieval signal. | |
| 741 | + $text = (string) file_get_contents($file['tmp_name']); | |
| 742 | + $text = wp_check_invalid_utf8($text); | |
| 743 | + $text = trim($text); | |
| 744 | + } | |
| 745 | + | |
| 746 | + if ($text === '') { | |
| 747 | + set_transient('mxchat_admin_notice_error', | |
| 748 | + esc_html__('The uploaded document contains no readable text.', 'mxchat'), | |
| 749 | + 30 | |
| 750 | + ); | |
| 751 | + wp_safe_redirect(esc_url($redirect_url)); | |
| 752 | + exit; | |
| 753 | + } | |
| 754 | + | |
| 755 | + // Size cap — same pdf_max_pages setting the PDF/toolbar paths use, but | |
| 756 | + // estimated by CHARACTERS (~2500/page): the .docx cleaner collapses all | |
| 757 | + // newlines to spaces, so a paragraph count reads 1 for any Word file. | |
| 758 | + // Processing is synchronous — an unbounded document risks a timeout. | |
| 759 | + $options = get_option('mxchat_options', array()); | |
| 760 | + $max_pages = isset($options['pdf_max_pages']) ? intval($options['pdf_max_pages']) : 69; | |
| 761 | + $estimated_pages = (int) ceil(strlen($text) / 2500); | |
| 762 | + if ($estimated_pages > $max_pages) { | |
| 763 | + set_transient('mxchat_admin_notice_error', | |
| 764 | + sprintf( | |
| 765 | + esc_html__('The document is too large (about %1$d pages; the limit is %2$d). Split it into smaller files, or raise the PDF max pages setting.', 'mxchat'), | |
| 766 | + $estimated_pages, | |
| 767 | + $max_pages | |
| 768 | + ), | |
| 769 | + 30 | |
| 770 | + ); | |
| 771 | + wp_safe_redirect(esc_url($redirect_url)); | |
| 772 | + exit; | |
| 773 | + } | |
| 774 | + | |
| 775 | + // Embedding API key — bot-aware, same shape as the direct-content handler. | |
| 776 | + $api_key = ''; | |
| 777 | + if ($bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager')) { | |
| 778 | + $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id); | |
| 779 | + $api_key = $bot_options['api_key'] ?? ''; | |
| 780 | + } | |
| 781 | + if (empty($api_key)) { | |
| 782 | + $api_key = $options['api_key'] ?? ''; | |
| 783 | + } | |
| 784 | + | |
| 785 | + // Stable identity — PDF Upload's scheme. A re-upload of the same filename | |
| 786 | + // replaces: clear old chunks first (covers a doc shrinking below the chunk | |
| 787 | + // threshold, where the single-vector path would not clean them), then | |
| 788 | + // submit — the chunked path re-deletes harmlessly. | |
| 789 | + $source_label = 'upload://' . $original_filename; | |
| 790 | + MxChat_Utils::delete_chunks_for_url($source_label, $bot_id); | |
| 791 | + $result = MxChat_Utils::submit_content_to_db($text, $source_label, $api_key, null, $bot_id, 'document'); | |
| 792 | + | |
| 793 | + if (is_wp_error($result)) { | |
| 794 | + set_transient('mxchat_admin_notice_error', | |
| 795 | + esc_html__('Failed to import the document: ', 'mxchat') . esc_html($result->get_error_message()), | |
| 796 | + 30 | |
| 797 | + ); | |
| 798 | + } else { | |
| 799 | + set_transient('mxchat_admin_notice_success', | |
| 800 | + sprintf( | |
| 801 | + esc_html__('Document "%s" imported into the knowledge base.', 'mxchat'), | |
| 802 | + esc_html($original_filename) | |
| 803 | + ), | |
| 804 | + 30 | |
| 805 | + ); | |
| 806 | + } | |
| 807 | + | |
| 808 | + wp_safe_redirect(esc_url($redirect_url)); | |
| 809 | + exit; | |
| 810 | +} | |
| 811 | + | |
| 812 | +/** | |
| 408 | 813 | * Validate PDF and count pages with multiple parser attempts |
| 409 | 814 | */ |
| 410 | 815 | private function mxchat_validate_and_count_pdf_pages($pdf_path) { |
| 411 | 816 | // Method 1: Try with Smalot PDF Parser (your current method) |
| @@ -628,8 +1033,25 @@ | ||
| 628 | 1033 | /** |
| 629 | 1034 | * AJAX: Get full content for editing — reassembles chunks if needed. |
| 630 | 1035 | * Works for both WordPress DB and Pinecone entries. |
| 631 | 1036 | */ |
| 1037 | +/** | |
| 1038 | + * Sanitize a knowledge entry's source_url from an AJAX request WITHOUT destroying | |
| 1039 | + * its identity. sanitize_text_field() strips percent-encoded octets (%20, %D7%A9…), | |
| 1040 | + * so a percent-encoded URL — every non-ASCII permalink — would md5 to a DIFFERENT | |
| 1041 | + * id than the one it was stored under: reads miss the entry and saves write an | |
| 1042 | + * orphan copy while the original keeps its stale text. URLs get esc_url_raw | |
| 1043 | + * (identity-preserving, matches what import stored); non-URL keys (mxchat://, | |
| 1044 | + * _ungrouped_) keep the old sanitizer. | |
| 1045 | + */ | |
| 1046 | +private function sanitize_entry_source_url( $raw ) { | |
| 1047 | + $raw = trim( (string) $raw ); | |
| 1048 | + if ( preg_match( '#^https?://#i', $raw ) ) { | |
| 1049 | + return esc_url_raw( $raw ); | |
| 1050 | + } | |
| 1051 | + return sanitize_text_field( $raw ); | |
| 1052 | +} | |
| 1053 | + | |
| 632 | 1054 | public function ajax_mxchat_get_entry_content() { |
| 633 | 1055 | check_ajax_referer('mxchat_edit_entry_nonce', 'nonce'); |
| 634 | 1056 | |
| 635 | 1057 | if ( ! current_user_can('manage_options') ) { |
| @@ -635,16 +1057,19 @@ | ||
| 635 | 1057 | if ( ! current_user_can('manage_options') ) { |
| 636 | 1058 | wp_send_json_error( array( 'message' => 'Permission denied.' ) ); |
| 637 | 1059 | } |
| 638 | 1060 | |
| 639 | - $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : ''; | |
| 1061 | + $source_url = $this->sanitize_entry_source_url( isset($_POST['source_url']) ? wp_unslash($_POST['source_url']) : '' ); | |
| 640 | 1062 | $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0; |
| 641 | 1063 | $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress'; |
| 642 | 1064 | $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default'; |
| 643 | 1065 | |
| 644 | 1066 | if ( $data_source === 'pinecone' ) { |
| 1067 | + // Pinecone ids are strings (md5 hashes, manual_* ids) — absint() would | |
| 1068 | + // destroy them, so re-read the raw value for this branch only. | |
| 1069 | + $vector_id = isset($_POST['entry_id']) ? sanitize_text_field( wp_unslash($_POST['entry_id']) ) : ''; | |
| 645 | 1070 | // Pinecone: fetch vectors by source_url, reassemble chunks |
| 646 | - $content = $this->get_pinecone_entry_content( $source_url, $entry_id, $bot_id ); | |
| 1071 | + $content = $this->get_pinecone_entry_content( $source_url, $vector_id, $bot_id ); | |
| 647 | 1072 | } else { |
| 648 | 1073 | // WordPress DB |
| 649 | 1074 | $content = $this->get_wordpress_entry_content( $source_url, $entry_id ); |
| 650 | 1075 | } |
| @@ -743,44 +1168,56 @@ | ||
| 743 | 1168 | if ( empty($host) || empty($api_key) ) { |
| 744 | 1169 | return new WP_Error( 'pinecone_config', 'Pinecone not configured.' ); |
| 745 | 1170 | } |
| 746 | 1171 | |
| 747 | - // List vectors with the source_url prefix | |
| 748 | - $base_id = md5( $source_url ); | |
| 749 | - $vector_ids = array( $base_id ); | |
| 1172 | + // Manual entries carry no source_url (their vector id is a minted manual_* string, | |
| 1173 | + // not md5 of anything the row can hand us) — fetch the exact vector instead. | |
| 1174 | + // '_ungrouped_' is the table view's synthetic display key for such rows. | |
| 1175 | + if ( ( empty($source_url) || strpos($source_url, '_ungrouped_') === 0 ) && ! empty($entry_id) && is_string($entry_id) ) { | |
| 1176 | + $vector_ids = array( $entry_id ); | |
| 1177 | + } else { | |
| 1178 | + // List vectors with the source_url prefix | |
| 1179 | + $base_id = md5( $source_url ); | |
| 1180 | + $vector_ids = array( $base_id ); | |
| 750 | 1181 | |
| 751 | - // Find chunk vectors | |
| 752 | - $list_url = "https://{$host}/vectors/list"; | |
| 753 | - $list_body = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 ); | |
| 754 | - if ( ! empty($namespace) ) { | |
| 755 | - $list_body['namespace'] = $namespace; | |
| 756 | - } | |
| 1182 | + // Find chunk vectors | |
| 1183 | + // NOTE: Pinecone's /vectors/list is a GET endpoint with query parameters. | |
| 1184 | + // A POST is answered 200-with-an-empty-body, which reads as "no vectors". | |
| 1185 | + $list_url = "https://{$host}/vectors/list"; | |
| 1186 | + $list_params = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 ); | |
| 1187 | + if ( ! empty($namespace) ) { | |
| 1188 | + $list_params['namespace'] = $namespace; | |
| 1189 | + } | |
| 757 | 1190 | |
| 758 | - $list_resp = wp_remote_post( $list_url, array( | |
| 759 | - 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ), | |
| 760 | - 'body' => wp_json_encode( $list_body ), | |
| 761 | - 'timeout' => 15, | |
| 762 | - ) ); | |
| 1191 | + $list_resp = wp_remote_get( $list_url . '?' . http_build_query( $list_params ), array( | |
| 1192 | + 'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ), | |
| 1193 | + 'timeout' => 15, | |
| 1194 | + ) ); | |
| 763 | 1195 | |
| 764 | - if ( ! is_wp_error($list_resp) ) { | |
| 765 | - $list_data = json_decode( wp_remote_retrieve_body($list_resp), true ); | |
| 766 | - if ( ! empty($list_data['vectors']) ) { | |
| 767 | - foreach ( $list_data['vectors'] as $v ) { | |
| 768 | - $vector_ids[] = $v['id']; | |
| 1196 | + if ( ! is_wp_error($list_resp) ) { | |
| 1197 | + $list_data = json_decode( wp_remote_retrieve_body($list_resp), true ); | |
| 1198 | + if ( ! empty($list_data['vectors']) ) { | |
| 1199 | + foreach ( $list_data['vectors'] as $v ) { | |
| 1200 | + $vector_ids[] = $v['id']; | |
| 1201 | + } | |
| 769 | 1202 | } |
| 770 | 1203 | } |
| 771 | 1204 | } |
| 772 | 1205 | |
| 773 | 1206 | // Fetch vectors with metadata |
| 774 | - $fetch_url = "https://{$host}/vectors/fetch"; | |
| 775 | - $fetch_body = array( 'ids' => $vector_ids ); | |
| 1207 | + // NOTE: /vectors/fetch is a GET endpoint too, and Pinecone expects the ids | |
| 1208 | + // repeated (ids=a&ids=b) — http_build_query would emit ids[0]=a, so build | |
| 1209 | + // the query string explicitly. | |
| 1210 | + $fetch_query = array(); | |
| 1211 | + foreach ( $vector_ids as $fetch_vid ) { | |
| 1212 | + $fetch_query[] = 'ids=' . rawurlencode( $fetch_vid ); | |
| 1213 | + } | |
| 776 | 1214 | if ( ! empty($namespace) ) { |
| 777 | - $fetch_body['namespace'] = $namespace; | |
| 1215 | + $fetch_query[] = 'namespace=' . rawurlencode( $namespace ); | |
| 778 | 1216 | } |
| 779 | 1217 | |
| 780 | - $fetch_resp = wp_remote_post( $fetch_url, array( | |
| 781 | - 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ), | |
| 782 | - 'body' => wp_json_encode( $fetch_body ), | |
| 1218 | + $fetch_resp = wp_remote_get( "https://{$host}/vectors/fetch?" . implode( '&', $fetch_query ), array( | |
| 1219 | + 'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ), | |
| 783 | 1220 | 'timeout' => 15, |
| 784 | 1221 | ) ); |
| 785 | 1222 | |
| 786 | 1223 | if ( is_wp_error($fetch_resp) ) { |
| @@ -947,17 +1384,18 @@ | ||
| 947 | 1384 | |
| 948 | 1385 | $base_id = md5( $source_url ); |
| 949 | 1386 | $vector_ids = array( $base_id ); |
| 950 | 1387 | |
| 951 | - $list_url = "https://{$host}/vectors/list"; | |
| 952 | - $list_body = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 ); | |
| 1388 | + // NOTE: Pinecone's /vectors/list is a GET endpoint with query parameters. | |
| 1389 | + // A POST is answered 200-with-an-empty-body, which reads as "no vectors". | |
| 1390 | + $list_url = "https://{$host}/vectors/list"; | |
| 1391 | + $list_params = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 ); | |
| 953 | 1392 | if ( ! empty($namespace) ) { |
| 954 | - $list_body['namespace'] = $namespace; | |
| 1393 | + $list_params['namespace'] = $namespace; | |
| 955 | 1394 | } |
| 956 | 1395 | |
| 957 | - $list_resp = wp_remote_post( $list_url, array( | |
| 958 | - 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ), | |
| 959 | - 'body' => wp_json_encode( $list_body ), | |
| 1396 | + $list_resp = wp_remote_get( $list_url . '?' . http_build_query( $list_params ), array( | |
| 1397 | + 'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ), | |
| 960 | 1398 | 'timeout' => 15, |
| 961 | 1399 | ) ); |
| 962 | 1400 | |
| 963 | 1401 | if ( ! is_wp_error($list_resp) ) { |
| @@ -968,17 +1406,21 @@ | ||
| 968 | 1406 | } |
| 969 | 1407 | } |
| 970 | 1408 | } |
| 971 | 1409 | |
| 972 | - $fetch_url = "https://{$host}/vectors/fetch"; | |
| 973 | - $fetch_body = array( 'ids' => $vector_ids ); | |
| 1410 | + // NOTE: /vectors/fetch is a GET endpoint too, and Pinecone expects the ids | |
| 1411 | + // repeated (ids=a&ids=b) — http_build_query would emit ids[0]=a, so build | |
| 1412 | + // the query string explicitly. | |
| 1413 | + $fetch_query = array(); | |
| 1414 | + foreach ( $vector_ids as $fetch_vid ) { | |
| 1415 | + $fetch_query[] = 'ids=' . rawurlencode( $fetch_vid ); | |
| 1416 | + } | |
| 974 | 1417 | if ( ! empty($namespace) ) { |
| 975 | - $fetch_body['namespace'] = $namespace; | |
| 1418 | + $fetch_query[] = 'namespace=' . rawurlencode( $namespace ); | |
| 976 | 1419 | } |
| 977 | 1420 | |
| 978 | - $fetch_resp = wp_remote_post( $fetch_url, array( | |
| 979 | - 'headers' => array( 'Api-Key' => $api_key, 'Content-Type' => 'application/json' ), | |
| 980 | - 'body' => wp_json_encode( $fetch_body ), | |
| 1421 | + $fetch_resp = wp_remote_get( "https://{$host}/vectors/fetch?" . implode( '&', $fetch_query ), array( | |
| 1422 | + 'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ), | |
| 981 | 1423 | 'timeout' => 15, |
| 982 | 1424 | ) ); |
| 983 | 1425 | |
| 984 | 1426 | if ( is_wp_error($fetch_resp) ) { |
| @@ -1047,9 +1489,9 @@ | ||
| 1047 | 1489 | if ( ! current_user_can('manage_options') ) { |
| 1048 | 1490 | wp_send_json_error( array( 'message' => 'Permission denied.' ) ); |
| 1049 | 1491 | } |
| 1050 | 1492 | |
| 1051 | - $source_url = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : ''; | |
| 1493 | + $source_url = $this->sanitize_entry_source_url( isset($_POST['source_url']) ? wp_unslash($_POST['source_url']) : '' ); | |
| 1052 | 1494 | $entry_id = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0; |
| 1053 | 1495 | $content = isset($_POST['content']) ? wp_kses_post( wp_unslash($_POST['content']) ) : ''; |
| 1054 | 1496 | $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress'; |
| 1055 | 1497 | $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default'; |
| @@ -1070,38 +1512,74 @@ | ||
| 1070 | 1512 | if ( empty($api_key) ) { |
| 1071 | 1513 | $api_key = $options['api_key'] ?? ''; |
| 1072 | 1514 | } |
| 1073 | 1515 | |
| 1074 | - global $wpdb; | |
| 1075 | - $table = $wpdb->prefix . 'mxchat_system_prompt_content'; | |
| 1516 | + if ( $data_source === 'pinecone' ) { | |
| 1517 | + // Pinecone branch. The WP-DB manual-entry delete below must never run here: | |
| 1518 | + // Pinecone ids are strings, and absint() on a digit-leading md5 hash would | |
| 1519 | + // yield a real (unrelated) WP row id. | |
| 1520 | + $raw_vector_id = isset($_POST['entry_id']) ? sanitize_text_field( wp_unslash($_POST['entry_id']) ) : ''; | |
| 1521 | + $is_manual_single = empty($source_url) || strpos($source_url, '_ungrouped_') === 0; | |
| 1522 | + $is_manual_chunked = strpos($source_url, 'mxchat://') === 0; | |
| 1076 | 1523 | |
| 1077 | - // If source_url is empty but we have an entry_id, look it up | |
| 1078 | - if ( empty($source_url) && $entry_id > 0 && $data_source === 'wordpress' ) { | |
| 1079 | - $row = $wpdb->get_row( $wpdb->prepare( "SELECT source_url FROM {$table} WHERE id = %d", $entry_id ) ); | |
| 1080 | - if ( $row && ! empty($row->source_url) ) { | |
| 1081 | - $source_url = $row->source_url; | |
| 1524 | + if ( $is_manual_single || $is_manual_chunked ) { | |
| 1525 | + // Manual content: remove the old vectors first, then store as fresh manual | |
| 1526 | + // content — submit_content_to_db mints a new unique identity (manual_* id | |
| 1527 | + // for a single vector, an mxchat:// chunk prefix if it now chunks). | |
| 1528 | + if ( $is_manual_chunked ) { | |
| 1529 | + // Minted identity: base + chunk vectors share the md5(mxchat://...) prefix. | |
| 1530 | + MxChat_Utils::delete_chunks_for_url( $source_url, $bot_id ); | |
| 1531 | + } elseif ( ! empty($raw_vector_id) && class_exists('MxChat_Pinecone_Manager') ) { | |
| 1532 | + $pinecone_manager = MxChat_Pinecone_Manager::get_instance(); | |
| 1533 | + $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options( $bot_id ); | |
| 1534 | + if ( ! empty($pinecone_options['mxchat_pinecone_api_key']) && ! empty($pinecone_options['mxchat_pinecone_host']) ) { | |
| 1535 | + $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id( | |
| 1536 | + $raw_vector_id, | |
| 1537 | + $pinecone_options['mxchat_pinecone_api_key'], | |
| 1538 | + $pinecone_options['mxchat_pinecone_host'], | |
| 1539 | + $pinecone_options['mxchat_pinecone_namespace'] ?? '' | |
| 1540 | + ); | |
| 1541 | + } | |
| 1542 | + } | |
| 1543 | + $result = MxChat_Utils::submit_content_to_db( $content, '', $api_key, null, $bot_id, $content_type ); | |
| 1544 | + } else { | |
| 1545 | + // URL-sourced entry: identity is md5(source_url). submit_content_to_db | |
| 1546 | + // handles delete-old-chunks → re-chunk → re-embed → store, and sweeps | |
| 1547 | + // stale chunk vectors when the content now fits in a single vector. | |
| 1548 | + $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, md5($source_url), $bot_id, $content_type ); | |
| 1082 | 1549 | } |
| 1083 | - } | |
| 1550 | + } else { | |
| 1551 | + global $wpdb; | |
| 1552 | + $table = $wpdb->prefix . 'mxchat_system_prompt_content'; | |
| 1084 | 1553 | |
| 1085 | - // For manual entries (no source_url or mxchat:// prefix), delete the old entry by ID first | |
| 1086 | - // so submit_content_to_db creates a replacement instead of a duplicate | |
| 1087 | - // Also treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries | |
| 1088 | - $is_legacy_manual = !empty($source_url) && strpos($source_url, 'mxchat.ai') !== false && strpos($source_url, 'mxchat://') !== 0; | |
| 1089 | - if ( $entry_id > 0 && (empty($source_url) || strpos($source_url, 'mxchat://') === 0 || $is_legacy_manual) ) { | |
| 1090 | - $wpdb->delete( $table, array( 'id' => $entry_id ), array( '%d' ) ); | |
| 1091 | - // Clear legacy URL so submit_content_to_db generates a unique mxchat:// identifier | |
| 1092 | - // instead of reusing the shared URL (which would mass-delete other entries with the same URL) | |
| 1093 | - if ( $is_legacy_manual ) { | |
| 1094 | - $source_url = ''; | |
| 1554 | + // If source_url is empty but we have an entry_id, look it up | |
| 1555 | + if ( empty($source_url) && $entry_id > 0 ) { | |
| 1556 | + $row = $wpdb->get_row( $wpdb->prepare( "SELECT source_url FROM {$table} WHERE id = %d", $entry_id ) ); | |
| 1557 | + if ( $row && ! empty($row->source_url) ) { | |
| 1558 | + $source_url = $row->source_url; | |
| 1559 | + } | |
| 1095 | 1560 | } |
| 1096 | - } | |
| 1097 | 1561 | |
| 1098 | - // Use the existing submit_content_to_db which handles chunking, Pinecone, and WP DB | |
| 1099 | - $vector_id = ! empty($source_url) ? md5($source_url) : md5('mxchat_manual_' . $entry_id); | |
| 1562 | + // For manual entries (no source_url or mxchat:// prefix), delete the old entry by ID first | |
| 1563 | + // so submit_content_to_db creates a replacement instead of a duplicate | |
| 1564 | + // Also treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries | |
| 1565 | + $is_legacy_manual = !empty($source_url) && strpos($source_url, 'mxchat.ai') !== false && strpos($source_url, 'mxchat://') !== 0; | |
| 1566 | + if ( $entry_id > 0 && (empty($source_url) || strpos($source_url, 'mxchat://') === 0 || $is_legacy_manual) ) { | |
| 1567 | + $wpdb->delete( $table, array( 'id' => $entry_id ), array( '%d' ) ); | |
| 1568 | + // Clear legacy URL so submit_content_to_db generates a unique mxchat:// identifier | |
| 1569 | + // instead of reusing the shared URL (which would mass-delete other entries with the same URL) | |
| 1570 | + if ( $is_legacy_manual ) { | |
| 1571 | + $source_url = ''; | |
| 1572 | + } | |
| 1573 | + } | |
| 1100 | 1574 | |
| 1101 | - // submit_content_to_db already handles: delete old chunks → re-chunk → re-embed → store | |
| 1102 | - $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, $vector_id, $bot_id, $content_type ); | |
| 1575 | + // Use the existing submit_content_to_db which handles chunking, Pinecone, and WP DB | |
| 1576 | + $vector_id = ! empty($source_url) ? md5($source_url) : md5('mxchat_manual_' . $entry_id); | |
| 1103 | 1577 | |
| 1578 | + // submit_content_to_db already handles: delete old chunks → re-chunk → re-embed → store | |
| 1579 | + $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, $vector_id, $bot_id, $content_type ); | |
| 1580 | + } | |
| 1581 | + | |
| 1104 | 1582 | if ( is_wp_error($result) ) { |
| 1105 | 1583 | wp_send_json_error( array( 'message' => $result->get_error_message() ) ); |
| 1106 | 1584 | } |
| 1107 | 1585 | |
| @@ -1187,42 +1665,34 @@ | ||
| 1187 | 1665 | |
| 1188 | 1666 | // Get bot_id from form submission |
| 1189 | 1667 | $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default'; |
| 1190 | 1668 | |
| 1191 | - // Get bot-specific options and validate API key | |
| 1669 | + // Get bot-specific options and validate the embedding decision — | |
| 1670 | + // custom-provider-aware (plan cbd5fd). | |
| 1192 | 1671 | $bot_options = $this->get_bot_options($bot_id); |
| 1193 | 1672 | $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options'); |
| 1194 | - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002'; | |
| 1195 | - | |
| 1196 | - if (strpos($selected_model, 'voyage') === 0) { | |
| 1197 | - $api_key = $options['voyage_api_key'] ?? ''; | |
| 1198 | - $provider_name = 'Voyage AI'; | |
| 1199 | - } elseif (strpos($selected_model, 'gemini-embedding') === 0) { | |
| 1200 | - $api_key = $options['gemini_api_key'] ?? ''; | |
| 1201 | - $provider_name = 'Google Gemini'; | |
| 1202 | - } else { | |
| 1203 | - $api_key = $options['api_key'] ?? ''; | |
| 1204 | - $provider_name = 'OpenAI'; | |
| 1205 | - } | |
| 1206 | - | |
| 1207 | - if (empty($api_key)) { | |
| 1208 | - $error_message = sprintf( | |
| 1209 | - esc_html__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'), | |
| 1210 | - $provider_name | |
| 1211 | - ); | |
| 1212 | - set_transient('mxchat_admin_notice_error', $error_message, 30); | |
| 1673 | + | |
| 1674 | + $preflight = MxChat_Utils::embedding_preflight($options); | |
| 1675 | + if (!$preflight['ok']) { | |
| 1676 | + set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30); | |
| 1213 | 1677 | wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts'))); |
| 1214 | 1678 | exit; |
| 1215 | 1679 | } |
| 1680 | + $api_key = $preflight['api_key']; | |
| 1216 | 1681 | |
| 1217 | - // Fetch URL — use browser-like headers so servers with bot protection don't block us | |
| 1682 | + // Fetch URL — send an honest, versioned MXChat crawler UA (not a spoofed | |
| 1683 | + // browser). Stale browser UAs are exactly what WAFs like SiteGround's | |
| 1684 | + // ModSecurity flag as scrapers, 403-ing the fetch (including PDFs served | |
| 1685 | + // from the site's own media library, which route through this same call). | |
| 1686 | + // See mxchat_ingest_user_agent(). Accept is kept for content negotiation; | |
| 1687 | + // the browser-only Accept-Language fingerprint is dropped so it stays | |
| 1688 | + // coherent with a bot identity. | |
| 1218 | 1689 | $response = wp_remote_get($submitted_url, array( |
| 1219 | 1690 | 'timeout' => 30, |
| 1220 | 1691 | 'sslverify' => false, |
| 1221 | - 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', | |
| 1692 | + 'user-agent' => mxchat_ingest_user_agent(), | |
| 1222 | 1693 | 'headers' => array( |
| 1223 | 1694 | 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', |
| 1224 | - 'Accept-Language' => 'en-US,en;q=0.9', | |
| 1225 | 1695 | ), |
| 1226 | 1696 | )); |
| 1227 | 1697 | |
| 1228 | 1698 | if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) { |
| @@ -1293,12 +1763,22 @@ | ||
| 1293 | 1763 | esc_html__('Sitemap queued for processing. Processing will start automatically.', 'mxchat'), |
| 1294 | 1764 | 30 |
| 1295 | 1765 | ); |
| 1296 | 1766 | } else { |
| 1297 | - set_transient('mxchat_admin_notice_error', | |
| 1298 | - esc_html__('Failed to queue sitemap processing. Please check the status below for details.', 'mxchat'), | |
| 1299 | - 30 | |
| 1300 | - ); | |
| 1767 | + // Surface the reason the handler already computed (embedding pre-flight, | |
| 1768 | + // empty sitemap, queue failure). The old message pointed at the status | |
| 1769 | + // area, which is empty on this path — nothing was ever queued. | |
| 1770 | + if (is_string($result) && $result !== '') { | |
| 1771 | + set_transient('mxchat_admin_notice_error', | |
| 1772 | + esc_html__('Failed to queue sitemap processing: ', 'mxchat') . esc_html($result), | |
| 1773 | + 30 | |
| 1774 | + ); | |
| 1775 | + } else { | |
| 1776 | + set_transient('mxchat_admin_notice_error', | |
| 1777 | + esc_html__('Failed to queue sitemap processing.', 'mxchat'), | |
| 1778 | + 30 | |
| 1779 | + ); | |
| 1780 | + } | |
| 1301 | 1781 | } |
| 1302 | 1782 | |
| 1303 | 1783 | wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts'))); |
| 1304 | 1784 | exit; |
| @@ -1441,8 +1921,140 @@ | ||
| 1441 | 1921 | * |
| 1442 | 1922 | * @param string $content The content containing shortcodes |
| 1443 | 1923 | * @return string Content with shortcode tags removed but inner content preserved |
| 1444 | 1924 | */ |
| 1925 | +/** | |
| 1926 | + * Single-pass HTML entity decode for text entering the knowledge base. | |
| 1927 | + * The corpus should hold what a human reads: a stored `&` consumes | |
| 1928 | + * extra tokens, distorts the vector away from the form a visitor's | |
| 1929 | + * question uses, and can be quoted back verbatim in an answer. | |
| 1930 | + * Deliberately NOT looped to a fixed point — a stored `&amp;` is a | |
| 1931 | + * legitimate literal `&` and must not collapse further (data loss). | |
| 1932 | + * UTF-8 charset keeps multibyte (CJK/RTL) text untouched. Both assembly | |
| 1933 | + * paths call this at their output points so the treatment cannot drift. | |
| 1934 | + * (Plan d2c92e.) | |
| 1935 | + */ | |
| 1936 | +private function mxchat_decode_entities_for_indexing($text) { | |
| 1937 | + return html_entity_decode((string) $text, ENT_QUOTES | ENT_HTML5, 'UTF-8'); | |
| 1938 | +} | |
| 1939 | + | |
| 1940 | +/** | |
| 1941 | + * Price lines for a product's indexed text, pinned to the store's BASE currency. | |
| 1942 | + * | |
| 1943 | + * The four product assembly paths each used to call get_woocommerce_currency_symbol() | |
| 1944 | + * with no argument, which resolves the currency active on the CURRENT request. | |
| 1945 | + * Multi-currency plugins (CURCY, WOOCS, Aelia, WPML Multicurrency) filter that per | |
| 1946 | + * request, so whichever currency the store happened to be serving when an import ran | |
| 1947 | + * was frozen into every product it indexed. The amounts have the mirror problem: the | |
| 1948 | + * woocommerce_product_get_* filters convert prices in the 'view' context but not in | |
| 1949 | + * 'edit', so a converted amount could be paired with an unconverted symbol and produce | |
| 1950 | + * a price that is not merely wrong but incoherent. | |
| 1951 | + * | |
| 1952 | + * Base currency option + 'edit' context makes both halves agree and makes the output | |
| 1953 | + * independent of when the import ran. The currency CODE is emitted alongside the symbol | |
| 1954 | + * so the model cannot read a bare "$" as USD on a store whose plugin swapped symbols. | |
| 1955 | + * (Plan 7403ec.) | |
| 1956 | + */ | |
| 1957 | +private function mxchat_product_price_lines($product) { | |
| 1958 | + if (!is_object($product) || !method_exists($product, 'get_regular_price')) { | |
| 1959 | + return ''; | |
| 1960 | + } | |
| 1961 | + | |
| 1962 | + $currency = get_option('woocommerce_currency'); | |
| 1963 | + $currency = is_string($currency) ? trim($currency) : ''; | |
| 1964 | + $symbol = ($currency !== '') | |
| 1965 | + ? get_woocommerce_currency_symbol($currency) | |
| 1966 | + : get_woocommerce_currency_symbol(); | |
| 1967 | + $symbol = $this->mxchat_decode_entities_for_indexing($symbol); | |
| 1968 | + | |
| 1969 | + $regular_price = $product->get_regular_price('edit'); | |
| 1970 | + $sale_price = $product->get_sale_price('edit'); | |
| 1971 | + $price = $product->get_price('edit'); | |
| 1972 | + | |
| 1973 | + $lines = ''; | |
| 1974 | + | |
| 1975 | + if (!empty($regular_price)) { | |
| 1976 | + $lines .= "Price: " . $this->mxchat_format_indexed_price($regular_price, $currency, $symbol) . "\n"; | |
| 1977 | + } elseif (!empty($price)) { | |
| 1978 | + $lines .= "Price: " . $this->mxchat_format_indexed_price($price, $currency, $symbol) . "\n"; | |
| 1979 | + } | |
| 1980 | + | |
| 1981 | + if (!empty($sale_price) && $sale_price !== $regular_price) { | |
| 1982 | + $lines .= "Sale Price: " . $this->mxchat_format_indexed_price($sale_price, $currency, $symbol) . "\n"; | |
| 1983 | + } | |
| 1984 | + | |
| 1985 | + if ($product->is_type('variable')) { | |
| 1986 | + list($min_price, $max_price) = $this->mxchat_variation_price_range($product); | |
| 1987 | + if ($min_price !== null && $max_price !== null && (float) $min_price !== (float) $max_price) { | |
| 1988 | + $lines .= "Price Range: " . $this->mxchat_format_indexed_price($min_price, $currency, $symbol) | |
| 1989 | + . " - " . $this->mxchat_format_indexed_price($max_price, $currency, $symbol) . "\n"; | |
| 1990 | + } | |
| 1991 | + } | |
| 1992 | + | |
| 1993 | + return $lines; | |
| 1994 | +} | |
| 1995 | + | |
| 1996 | +/** | |
| 1997 | + * One indexed price amount, labelled with its currency code. | |
| 1998 | + * | |
| 1999 | + * "INR 1299.00 (Rs.1299.00)" — the code is what the model should reason from; the symbol | |
| 2000 | + * is kept so a quoted price still reads naturally. Falls back to the old symbol-only | |
| 2001 | + * shape when WooCommerce has no base currency configured, and drops the parenthetical | |
| 2002 | + * when the symbol is absent or IS the code (several currencies have no distinct glyph). | |
| 2003 | + */ | |
| 2004 | +private function mxchat_format_indexed_price($amount, $currency, $symbol) { | |
| 2005 | + $amount = (string) $amount; | |
| 2006 | + | |
| 2007 | + if ($currency === '') { | |
| 2008 | + return $symbol . $amount; | |
| 2009 | + } | |
| 2010 | + | |
| 2011 | + if ($symbol === '' || $symbol === $currency) { | |
| 2012 | + return $currency . ' ' . $amount; | |
| 2013 | + } | |
| 2014 | + | |
| 2015 | + return $currency . ' ' . $amount . ' (' . $symbol . $amount . ')'; | |
| 2016 | +} | |
| 2017 | + | |
| 2018 | +/** | |
| 2019 | + * Min/max variation price read from the variations themselves in 'edit' context. | |
| 2020 | + * | |
| 2021 | + * get_variation_price() reads WooCommerce's display price cache, which multi-currency | |
| 2022 | + * plugins populate with converted values — the same defect the rest of this helper | |
| 2023 | + * exists to remove. Returns raw stored strings (not floats) so the indexed text keeps | |
| 2024 | + * the store's own price formatting, and (null, null) when no variation carries a price. | |
| 2025 | + */ | |
| 2026 | +private function mxchat_variation_price_range($product) { | |
| 2027 | + $min_raw = null; | |
| 2028 | + $max_raw = null; | |
| 2029 | + $min_val = null; | |
| 2030 | + $max_val = null; | |
| 2031 | + | |
| 2032 | + $children = method_exists($product, 'get_children') ? $product->get_children() : array(); | |
| 2033 | + | |
| 2034 | + foreach ($children as $child_id) { | |
| 2035 | + $variation = wc_get_product($child_id); | |
| 2036 | + if (!$variation) { | |
| 2037 | + continue; | |
| 2038 | + } | |
| 2039 | + $raw = $variation->get_price('edit'); | |
| 2040 | + if ($raw === '' || $raw === null) { | |
| 2041 | + continue; | |
| 2042 | + } | |
| 2043 | + $val = (float) $raw; | |
| 2044 | + if ($min_val === null || $val < $min_val) { | |
| 2045 | + $min_val = $val; | |
| 2046 | + $min_raw = $raw; | |
| 2047 | + } | |
| 2048 | + if ($max_val === null || $val > $max_val) { | |
| 2049 | + $max_val = $val; | |
| 2050 | + $max_raw = $raw; | |
| 2051 | + } | |
| 2052 | + } | |
| 2053 | + | |
| 2054 | + return array($min_raw, $max_raw); | |
| 2055 | +} | |
| 2056 | + | |
| 1445 | 2057 | private function strip_shortcode_tags_preserve_content($content) { |
| 1446 | 2058 | // Single-pass regex removes all shortcode brackets: [tag], [tag attr="val"], [/tag], [tag /] |
| 1447 | 2059 | // Content between tags is inherently preserved since only brackets are targeted |
| 1448 | 2060 | $result = preg_replace('/\[\/?\w[\w-]*[^\]]*\]/', '', $content); |
| @@ -1486,24 +2098,37 @@ | ||
| 1486 | 2098 | |
| 1487 | 2099 | // Ensure valid UTF-8 encoding |
| 1488 | 2100 | $content = wp_check_invalid_utf8($content); |
| 1489 | 2101 | |
| 1490 | - // Remove any extremely long strings without spaces (often garbage) | |
| 1491 | - $content = preg_replace('/\S{300,}/', ' ', $content); | |
| 1492 | - | |
| 1493 | - // Replace problematic characters that often cause database issues | |
| 1494 | - $content = preg_replace('/[\x{10000}-\x{10FFFF}]/u', '', $content); // Remove emoji and other high Unicode characters | |
| 1495 | - | |
| 1496 | - // Replace any remaining potentially problematic characters with spaces | |
| 1497 | - // BUT preserve newlines by temporarily replacing them | |
| 1498 | - $content = str_replace("\n", "NEWLINE_PLACEHOLDER", $content); | |
| 1499 | - $content = preg_replace('/[^\p{L}\p{N}\p{P}\p{Z}\p{Sm}]/u', ' ', $content); | |
| 1500 | - $content = str_replace("NEWLINE_PLACEHOLDER", "\n", $content); | |
| 1501 | - | |
| 1502 | - // Limit to reasonable length if needed | |
| 2102 | + // Remove extremely long runs with no whitespace (base64 blobs, minified JS). | |
| 2103 | + // Counts CHARACTERS (/u), and never strips a run containing characters from a | |
| 2104 | + // script written without spaces — Japanese, Chinese, Thai, Khmer, Lao, Myanmar — | |
| 2105 | + // where a normal paragraph is legitimately one unbroken run. | |
| 2106 | + $content = preg_replace_callback('/\S{300,}/u', function ($m) { | |
| 2107 | + return preg_match('/[\p{Han}\p{Hiragana}\p{Katakana}\p{Thai}\p{Khmer}\p{Lao}\p{Myanmar}]/u', $m[0]) ? $m[0] : ' '; | |
| 2108 | + }, $content); | |
| 2109 | + | |
| 2110 | + // Remove emoji/symbol blocks only — not the whole supplementary plane, which | |
| 2111 | + // also holds CJK Extension B ideographs used in real Chinese/Japanese names. | |
| 2112 | + // A ZWJ (U+200D) BETWEEN stripped pictographs is consumed with them, so a | |
| 2113 | + // family sequence like 👨👩👧 leaves no invisible zero-width residue behind | |
| 2114 | + // (the joiner between NON-emoji characters — Hindi conjuncts — is untouched). | |
| 2115 | + $content = preg_replace('/[\x{1F000}-\x{1F0FF}\x{1F300}-\x{1FAFF}](?:\x{200D}[\x{1F000}-\x{1F0FF}\x{1F300}-\x{1FAFF}])*/u', '', $content); | |
| 2116 | + | |
| 2117 | + // There is deliberately NO catch-all character allowlist here (plan 209e57; | |
| 2118 | + // one existed until 3.2.20). Every genuinely dangerous byte is already gone: | |
| 2119 | + // control characters, null bytes, invalid UTF-8 and the emoji blocks are all | |
| 2120 | + // stripped above. The allowlist's only remaining effect was to damage scripts | |
| 2121 | + // nobody thought to enumerate — Unicode Cf (Format) was missing, so it | |
| 2122 | + // replaced the zero-width joiner/non-joiner with spaces and silently split | |
| 2123 | + // Persian words (میروم → می روم) and broke Hindi conjuncts (क्ष → क् ष). | |
| 2124 | + // Do not add one back; the failure mode of an allowlist is exactly this. | |
| 2125 | + | |
| 2126 | + // Limit to reasonable length if needed (byte limit — MySQL TEXT is byte-sized, | |
| 2127 | + // but cut on a character boundary so a multibyte char is never split mid-sequence) | |
| 1503 | 2128 | $max_length = 65000; // Just under MySQL TEXT field limit |
| 1504 | 2129 | if (strlen($content) > $max_length) { |
| 1505 | - $content = substr($content, 0, $max_length); | |
| 2130 | + $content = mb_strcut($content, 0, $max_length, 'UTF-8'); | |
| 1506 | 2131 | } |
| 1507 | 2132 | |
| 1508 | 2133 | //error_log('[MXCHAT-SANITIZE] Sanitized content preview: ' . substr($content, 0, 500) . '...'); |
| 1509 | 2134 | return $content; |
| @@ -2341,10 +2966,19 @@ | ||
| 2341 | 2966 | <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span> |
| 2342 | 2967 | <?php endif; ?> |
| 2343 | 2968 | </td> |
| 2344 | 2969 | <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;"> |
| 2345 | - <?php if ($data_source !== 'pinecone') : ?> | |
| 2346 | 2970 | <button type="button" |
| 2971 | + class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-inspect-entry-btn" | |
| 2972 | + data-source-url="<?php echo esc_attr($source_url); ?>" | |
| 2973 | + data-entry-id="<?php echo esc_attr($first_prompt->id); ?>" | |
| 2974 | + data-data-source="<?php echo esc_attr($data_source); ?>" | |
| 2975 | + data-bot-id="<?php echo esc_attr($current_bot_id); ?>" | |
| 2976 | + data-nonce="<?php echo wp_create_nonce('mxchat_inspect_entry_nonce'); ?>" | |
| 2977 | + title="<?php esc_attr_e('View indexed content', 'mxchat'); ?>"> | |
| 2978 | + <span class="dashicons dashicons-visibility" style="font-size: 14px;"></span> | |
| 2979 | + </button> | |
| 2980 | + <button type="button" | |
| 2347 | 2981 | class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn" |
| 2348 | 2982 | data-source-url="<?php echo esc_attr($source_url); ?>" |
| 2349 | 2983 | data-entry-id="<?php echo esc_attr($first_prompt->id); ?>" |
| 2350 | 2984 | data-data-source="<?php echo esc_attr($data_source); ?>" |
| @@ -2352,9 +2986,8 @@ | ||
| 2352 | 2986 | data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>" |
| 2353 | 2987 | title="<?php esc_attr_e('Edit content', 'mxchat'); ?>"> |
| 2354 | 2988 | <span class="dashicons dashicons-edit" style="font-size: 14px;"></span> |
| 2355 | 2989 | </button> |
| 2356 | - <?php endif; ?> | |
| 2357 | 2990 | <button type="button" |
| 2358 | 2991 | class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group" |
| 2359 | 2992 | data-source-url="<?php echo esc_attr($source_url); ?>" |
| 2360 | 2993 | data-chunk-count="<?php echo esc_attr($chunk_count); ?>" |
| @@ -2486,9 +3119,29 @@ | ||
| 2486 | 3119 | <?php else : ?> |
| 2487 | 3120 | <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span> |
| 2488 | 3121 | <?php endif; ?> |
| 2489 | 3122 | </td> |
| 2490 | - <td style="padding: 12px 16px;"> | |
| 3123 | + <td style="padding: 12px 16px; white-space: nowrap;"> | |
| 3124 | + <button type="button" | |
| 3125 | + class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-inspect-entry-btn" | |
| 3126 | + data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>" | |
| 3127 | + data-entry-id="<?php echo esc_attr($prompt->id); ?>" | |
| 3128 | + data-data-source="<?php echo esc_attr($data_source); ?>" | |
| 3129 | + data-bot-id="<?php echo esc_attr($current_bot_id); ?>" | |
| 3130 | + data-nonce="<?php echo wp_create_nonce('mxchat_inspect_entry_nonce'); ?>" | |
| 3131 | + title="<?php esc_attr_e('View indexed content', 'mxchat'); ?>"> | |
| 3132 | + <span class="dashicons dashicons-visibility" style="font-size: 14px;"></span> | |
| 3133 | + </button> | |
| 3134 | + <button type="button" | |
| 3135 | + class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn" | |
| 3136 | + data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>" | |
| 3137 | + data-entry-id="<?php echo esc_attr($prompt->id); ?>" | |
| 3138 | + data-data-source="<?php echo esc_attr($data_source); ?>" | |
| 3139 | + data-bot-id="<?php echo esc_attr($current_bot_id); ?>" | |
| 3140 | + data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>" | |
| 3141 | + title="<?php esc_attr_e('Edit content', 'mxchat'); ?>"> | |
| 3142 | + <span class="dashicons dashicons-edit" style="font-size: 14px;"></span> | |
| 3143 | + </button> | |
| 2491 | 3144 | <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-ajax" data-vector-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_pinecone_prompt_nonce'); ?>" style="color: var(--mxch-error);"> |
| 2492 | 3145 | <span class="dashicons dashicons-trash" style="font-size: 14px;"></span> |
| 2493 | 3146 | </button> |
| 2494 | 3147 | </td> |
| @@ -3061,9 +3714,9 @@ | ||
| 3061 | 3714 | $response = wp_remote_head($url, array( |
| 3062 | 3715 | 'timeout' => 10, |
| 3063 | 3716 | 'sslverify' => false, |
| 3064 | 3717 | 'redirection' => 1, |
| 3065 | - 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', | |
| 3718 | + 'user-agent' => mxchat_ingest_user_agent(), | |
| 3066 | 3719 | )); |
| 3067 | 3720 | |
| 3068 | 3721 | if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) { |
| 3069 | 3722 | // Found a sitemap index - parse it to get sub-sitemaps |
| @@ -3123,12 +3776,11 @@ | ||
| 3123 | 3776 | |
| 3124 | 3777 | $response = wp_remote_get($url, array( |
| 3125 | 3778 | 'timeout' => 30, |
| 3126 | 3779 | 'sslverify' => false, |
| 3127 | - 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', | |
| 3780 | + 'user-agent' => mxchat_ingest_user_agent(), | |
| 3128 | 3781 | 'headers' => array( |
| 3129 | 3782 | 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', |
| 3130 | - 'Accept-Language' => 'en-US,en;q=0.9', | |
| 3131 | 3783 | ), |
| 3132 | 3784 | )); |
| 3133 | 3785 | |
| 3134 | 3786 | if (is_wp_error($response)) { |
| @@ -3182,12 +3834,11 @@ | ||
| 3182 | 3834 | private function get_sitemap_url_count($url) { |
| 3183 | 3835 | $response = wp_remote_get($url, array( |
| 3184 | 3836 | 'timeout' => 30, |
| 3185 | 3837 | 'sslverify' => false, |
| 3186 | - 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', | |
| 3838 | + 'user-agent' => mxchat_ingest_user_agent(), | |
| 3187 | 3839 | 'headers' => array( |
| 3188 | 3840 | 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', |
| 3189 | - 'Accept-Language' => 'en-US,en;q=0.9', | |
| 3190 | 3841 | ), |
| 3191 | 3842 | )); |
| 3192 | 3843 | |
| 3193 | 3844 | if (is_wp_error($response)) { |
| @@ -3213,9 +3864,9 @@ | ||
| 3213 | 3864 | |
| 3214 | 3865 | $response = wp_remote_get($robots_url, array( |
| 3215 | 3866 | 'timeout' => 15, |
| 3216 | 3867 | 'sslverify' => false, |
| 3217 | - 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', | |
| 3868 | + 'user-agent' => mxchat_ingest_user_agent(), | |
| 3218 | 3869 | )); |
| 3219 | 3870 | |
| 3220 | 3871 | if (is_wp_error($response)) { |
| 3221 | 3872 | return $sitemaps; |
| @@ -3706,20 +4357,12 @@ | ||
| 3706 | 4357 | |
| 3707 | 4358 | // Get bot_id from request |
| 3708 | 4359 | $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default'; |
| 3709 | 4360 | |
| 3710 | - // ACF→PDF extraction is opt-in per import batch. Persist the last-used value so users | |
| 3711 | - // don't re-check on every batch; the default is OFF for installs that haven't set it. | |
| 3712 | - $extract_acf_pdfs = !empty($_POST['extract_acf_pdfs']) && $_POST['extract_acf_pdfs'] !== 'false'; | |
| 3713 | - $mxchat_options = get_option('mxchat_options', array()); | |
| 3714 | - if (!is_array($mxchat_options)) { | |
| 3715 | - $mxchat_options = array(); | |
| 3716 | - } | |
| 3717 | - $prior_default = !empty($mxchat_options['acf_pdf_extract_default']); | |
| 3718 | - if ($prior_default !== $extract_acf_pdfs) { | |
| 3719 | - $mxchat_options['acf_pdf_extract_default'] = $extract_acf_pdfs ? 1 : 0; | |
| 3720 | - update_option('mxchat_options', $mxchat_options); | |
| 3721 | - } | |
| 4361 | + // ACF→PDF extraction is an install-level setting (Knowledge → ACF Fields, | |
| 4362 | + // plan 11720c). The import modal shows a passive status line pointing | |
| 4363 | + // there; the old per-batch checkbox and its remembered default are gone. | |
| 4364 | + $extract_acf_pdfs = get_option('mxchat_acf_pdf_extraction', '0') === '1'; | |
| 3722 | 4365 | |
| 3723 | 4366 | // Process only ONE post at a time to avoid request size issues |
| 3724 | 4367 | $post_id = reset($post_ids); |
| 3725 | 4368 | $post = get_post($post_id); |
| @@ -3728,173 +4371,34 @@ | ||
| 3728 | 4371 | wp_send_json_error('Post not found'); |
| 3729 | 4372 | exit; |
| 3730 | 4373 | } |
| 3731 | 4374 | |
| 3732 | - // Allow developers to modify post data before processing into knowledge base | |
| 4375 | + /** | |
| 4376 | + * Allow developers to modify post data before processing into the knowledge base. | |
| 4377 | + * Applied on BOTH content-preparation paths (this manual bulk import and the | |
| 4378 | + * auto-sync path in mxchat_handle_post_update) with the same signature, so a | |
| 4379 | + * callback registered once covers every indexing route. Purely additive — | |
| 4380 | + * zero behaviour change when unhooked. | |
| 4381 | + * | |
| 4382 | + * @param WP_Post $post The post about to be indexed. | |
| 4383 | + * @param string $bot_id Bot context for this import. | |
| 4384 | + */ | |
| 3733 | 4385 | $post = apply_filters('mxchat_before_process_post', $post, $bot_id); |
| 3734 | - | |
| 3735 | - // Get content including title, short description (for WooCommerce), and main content | |
| 3736 | - $content = $post->post_title . "\n\n"; | |
| 3737 | - | |
| 3738 | - // Add short description if it exists (WooCommerce products use post_excerpt for short description) | |
| 3739 | - if (!empty($post->post_excerpt)) { | |
| 3740 | - // Remove shortcode tags but preserve content inside them | |
| 3741 | - $clean_excerpt = $this->strip_shortcode_tags_preserve_content($post->post_excerpt); | |
| 3742 | - $content .= "Short Description: " . wp_strip_all_tags($clean_excerpt) . "\n\n"; | |
| 4386 | + if (!($post instanceof WP_Post)) { | |
| 4387 | + $post = get_post($post_id); // defend against a bad callback return | |
| 3743 | 4388 | } |
| 3744 | 4389 | |
| 3745 | - // Add main content - remove shortcode tags but preserve content inside them | |
| 3746 | - $clean_content = $this->strip_shortcode_tags_preserve_content($post->post_content); | |
| 3747 | - $content .= wp_strip_all_tags($clean_content); | |
| 4390 | + // Assemble the indexable text via the shared post-kind assembler (a3d60c). | |
| 4391 | + // Bulk import reads raw post fields, has always included product custom tabs, | |
| 4392 | + // and passes the install-level ACF→PDF option. | |
| 4393 | + $prepared = $this->mxchat_prepare_post_content_for_indexing($post_id, $post, array( | |
| 4394 | + 'read_display' => false, | |
| 4395 | + 'extract_acf_pdfs' => $extract_acf_pdfs, | |
| 4396 | + 'include_product_tabs' => true, | |
| 4397 | + )); | |
| 4398 | + $content = $prepared['content']; | |
| 4399 | + $pdf_extracted_count = $prepared['pdf_extracted_count']; | |
| 3748 | 4400 | |
| 3749 | - // ADD WOOCOMMERCE PRODUCT DATA (pricing, stock, categories, custom tabs) | |
| 3750 | - if (get_post_type($post_id) === 'product' && class_exists('WooCommerce')) { | |
| 3751 | - $product = wc_get_product($post_id); | |
| 3752 | - | |
| 3753 | - if ($product) { | |
| 3754 | - // Get pricing information | |
| 3755 | - $regular_price = $product->get_regular_price(); | |
| 3756 | - $sale_price = $product->get_sale_price(); | |
| 3757 | - $price = $product->get_price(); | |
| 3758 | - $sku = $product->get_sku(); | |
| 3759 | - | |
| 3760 | - // Get currency symbol | |
| 3761 | - $currency_symbol = get_woocommerce_currency_symbol(); | |
| 3762 | - | |
| 3763 | - // Add pricing information | |
| 3764 | - $content .= "\n"; | |
| 3765 | - if (!empty($regular_price)) { | |
| 3766 | - $content .= "Price: " . $currency_symbol . $regular_price . "\n"; | |
| 3767 | - } elseif (!empty($price)) { | |
| 3768 | - $content .= "Price: " . $currency_symbol . $price . "\n"; | |
| 3769 | - } | |
| 3770 | - | |
| 3771 | - if (!empty($sale_price) && $sale_price !== $regular_price) { | |
| 3772 | - $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n"; | |
| 3773 | - } | |
| 3774 | - | |
| 3775 | - // Handle variable products - show price range | |
| 3776 | - if ($product->is_type('variable')) { | |
| 3777 | - $min_price = $product->get_variation_price('min'); | |
| 3778 | - $max_price = $product->get_variation_price('max'); | |
| 3779 | - if ($min_price !== $max_price) { | |
| 3780 | - $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n"; | |
| 3781 | - } | |
| 3782 | - } | |
| 3783 | - | |
| 3784 | - if (!empty($sku)) { | |
| 3785 | - $content .= "SKU: " . $sku . "\n"; | |
| 3786 | - } | |
| 3787 | - | |
| 3788 | - // Get product categories | |
| 3789 | - $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names')); | |
| 3790 | - if (!empty($categories) && !is_wp_error($categories)) { | |
| 3791 | - $content .= "Categories: " . implode(', ', $categories) . "\n"; | |
| 3792 | - } | |
| 3793 | - } | |
| 3794 | - | |
| 3795 | - // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots) | |
| 3796 | - $custom_tabs = get_post_meta($post_id, 'yikes_woo_products_tabs', true); | |
| 3797 | - if (!empty($custom_tabs) && is_array($custom_tabs)) { | |
| 3798 | - foreach ($custom_tabs as $tab) { | |
| 3799 | - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : ''); | |
| 3800 | - $tab_content = isset($tab['content']) ? $tab['content'] : ''; | |
| 3801 | - | |
| 3802 | - if (!empty($tab_title) && !empty($tab_content)) { | |
| 3803 | - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n"; | |
| 3804 | - } | |
| 3805 | - } | |
| 3806 | - } | |
| 3807 | - | |
| 3808 | - // Also check for reusable/saved tabs applied to this product | |
| 3809 | - $applied_saved_tabs = get_post_meta($post_id, 'yikes_woo_reusable_products_tabs_applied', true); | |
| 3810 | - if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) { | |
| 3811 | - $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array()); | |
| 3812 | - if (!empty($saved_tabs) && is_array($saved_tabs)) { | |
| 3813 | - foreach ($applied_saved_tabs as $saved_tab_id) { | |
| 3814 | - if (isset($saved_tabs[$saved_tab_id])) { | |
| 3815 | - $tab = $saved_tabs[$saved_tab_id]; | |
| 3816 | - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : ''); | |
| 3817 | - $tab_content = isset($tab['content']) ? $tab['content'] : ''; | |
| 3818 | - | |
| 3819 | - if (!empty($tab_title) && !empty($tab_content)) { | |
| 3820 | - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n"; | |
| 3821 | - } | |
| 3822 | - } | |
| 3823 | - } | |
| 3824 | - } | |
| 3825 | - } | |
| 3826 | - } | |
| 3827 | - | |
| 3828 | - // ADD ACF FIELDS SUPPORT | |
| 3829 | - $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id); | |
| 3830 | - $pdf_extracted_count = 0; | |
| 3831 | - if (!empty($acf_fields)) { | |
| 3832 | - $acf_content_parts = array(); | |
| 3833 | - $pdf_attachment_ids = array(); | |
| 3834 | - | |
| 3835 | - foreach ($acf_fields as $field_name => $field_value) { | |
| 3836 | - $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id); | |
| 3837 | - | |
| 3838 | - if (!empty($formatted_value)) { | |
| 3839 | - $field_label = ucwords(str_replace('_', ' ', $field_name)); | |
| 3840 | - $acf_content_parts[] = $field_label . ": " . $formatted_value; | |
| 3841 | - } | |
| 3842 | - | |
| 3843 | - // Walk this field's value tree for any PDF attachment references and queue them for extraction. | |
| 3844 | - // Only when the user opted into ACF→PDF extraction for this batch; otherwise the ACF text | |
| 3845 | - // still lands in the KB but the heavier PDF parsing is skipped. | |
| 3846 | - if ($extract_acf_pdfs) { | |
| 3847 | - $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids); | |
| 3848 | - } | |
| 3849 | - } | |
| 3850 | - | |
| 3851 | - if (!empty($acf_content_parts)) { | |
| 3852 | - $content .= "\n\n" . implode("\n", $acf_content_parts); | |
| 3853 | - } | |
| 3854 | - | |
| 3855 | - // Extract text from each unique PDF found in ACF fields and append as a labeled section | |
| 3856 | - if ($extract_acf_pdfs && !empty($pdf_attachment_ids)) { | |
| 3857 | - $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids))); | |
| 3858 | - $pdf_sections = array(); | |
| 3859 | - foreach ($pdf_attachment_ids as $att_id) { | |
| 3860 | - $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id); | |
| 3861 | - if (!empty($pdf_text)) { | |
| 3862 | - $pdf_title = get_the_title($att_id); | |
| 3863 | - $pdf_url = wp_get_attachment_url($att_id); | |
| 3864 | - $header = 'PDF Attachment'; | |
| 3865 | - if (!empty($pdf_title)) { | |
| 3866 | - $header .= ': ' . $pdf_title; | |
| 3867 | - } | |
| 3868 | - if (!empty($pdf_url)) { | |
| 3869 | - $header .= ' (' . $pdf_url . ')'; | |
| 3870 | - } | |
| 3871 | - $pdf_sections[] = $header . "\n" . $pdf_text; | |
| 3872 | - $pdf_extracted_count++; | |
| 3873 | - } | |
| 3874 | - } | |
| 3875 | - if (!empty($pdf_sections)) { | |
| 3876 | - $content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections); | |
| 3877 | - } | |
| 3878 | - } | |
| 3879 | - } | |
| 3880 | - | |
| 3881 | - // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields) | |
| 3882 | - $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id); | |
| 3883 | - if (!empty($custom_meta)) { | |
| 3884 | - $meta_content_parts = array(); | |
| 3885 | - | |
| 3886 | - foreach ($custom_meta as $meta_key => $meta_value) { | |
| 3887 | - // Convert meta key to readable label | |
| 3888 | - $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key)); | |
| 3889 | - $meta_content_parts[] = $meta_label . ": " . $meta_value; | |
| 3890 | - } | |
| 3891 | - | |
| 3892 | - if (!empty($meta_content_parts)) { | |
| 3893 | - $content .= "\n\n" . implode("\n", $meta_content_parts); | |
| 3894 | - } | |
| 3895 | - } | |
| 3896 | - | |
| 3897 | 4401 | // Debug logging for WordPress Import content |
| 3898 | 4402 | //error_log('[MXCHAT-WP-IMPORT-DEBUG] Post ID: ' . $post_id . ' Title: ' . $post->post_title); |
| 3899 | 4403 | //error_log('[MXCHAT-WP-IMPORT-DEBUG] Raw post_content length: ' . strlen($post->post_content)); |
| 3900 | 4404 | //error_log('[MXCHAT-WP-IMPORT-DEBUG] Final content length: ' . strlen($content)); |
| @@ -3901,29 +4405,19 @@ | ||
| 3901 | 4405 | //error_log('[MXCHAT-WP-IMPORT-DEBUG] Content preview: ' . substr($content, 0, 300)); |
| 3902 | 4406 | |
| 3903 | 4407 | // Note: Removed 10,000 char limit - chunking now handles large content properly |
| 3904 | 4408 | |
| 3905 | - // Get bot-specific API key | |
| 4409 | + // Get bot-specific embedding decision — custom-provider-aware (plan cbd5fd) | |
| 3906 | 4410 | $bot_options = $this->get_bot_options($bot_id); |
| 3907 | 4411 | $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options'); |
| 3908 | - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002'; | |
| 3909 | - | |
| 3910 | - if (strpos($selected_model, 'voyage') === 0) { | |
| 3911 | - $api_key = $options['voyage_api_key'] ?? ''; | |
| 3912 | - $provider_name = 'Voyage AI'; | |
| 3913 | - } elseif (strpos($selected_model, 'gemini-embedding') === 0) { | |
| 3914 | - $api_key = $options['gemini_api_key'] ?? ''; | |
| 3915 | - $provider_name = 'Google Gemini'; | |
| 3916 | - } else { | |
| 3917 | - $api_key = $options['api_key'] ?? ''; | |
| 3918 | - $provider_name = 'OpenAI'; | |
| 3919 | - } | |
| 3920 | - | |
| 3921 | - if (empty($api_key)) { | |
| 3922 | - MxChat_Admin::mxchat_log_debug('api_error', $provider_name . ' API key not configured for knowledge processing'); | |
| 3923 | - wp_send_json_error($provider_name . ' API key not configured'); | |
| 4412 | + | |
| 4413 | + $preflight = MxChat_Utils::embedding_preflight($options); | |
| 4414 | + if (!$preflight['ok']) { | |
| 4415 | + MxChat_Admin::mxchat_log_debug('api_error', $preflight['reason'] . ' (knowledge processing)'); | |
| 4416 | + wp_send_json_error($preflight['reason']); | |
| 3924 | 4417 | exit; |
| 3925 | 4418 | } |
| 4419 | + $api_key = $preflight['api_key']; | |
| 3926 | 4420 | |
| 3927 | 4421 | $source_url = get_permalink($post_id); |
| 3928 | 4422 | $vector_id = md5($source_url); // Vector ID for Pinecone |
| 3929 | 4423 | |
| @@ -3994,11 +4488,11 @@ | ||
| 3994 | 4488 | // Automatically apply role restriction based on tags |
| 3995 | 4489 | $this->apply_role_restriction_to_post($post_id, $source_url); |
| 3996 | 4490 | |
| 3997 | 4491 | $operation_type = $is_update ? 'update' : 'new'; |
| 3998 | - | |
| 4492 | + | |
| 3999 | 4493 | // Count ACF fields for debugging |
| 4000 | - $acf_field_count = count($acf_fields); | |
| 4494 | + $acf_field_count = $prepared['acf_fields_found']; | |
| 4001 | 4495 | |
| 4002 | 4496 | // Success response with minimal data |
| 4003 | 4497 | wp_send_json_success(array( |
| 4004 | 4498 | 'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully', |
| @@ -4078,9 +4572,9 @@ | ||
| 4078 | 4572 | ); |
| 4079 | 4573 | } else { |
| 4080 | 4574 | // Update WordPress DB |
| 4081 | 4575 | $table_name = $wpdb->prefix . 'mxchat_system_prompt_content'; |
| 4082 | - | |
| 4576 | + | |
| 4083 | 4577 | $wpdb->update( |
| 4084 | 4578 | $table_name, |
| 4085 | 4579 | array('role_restriction' => $highest_role), |
| 4086 | 4580 | array('source_url' => $source_url), |
| @@ -4087,8 +4581,15 @@ | ||
| 4087 | 4581 | array('%s'), |
| 4088 | 4582 | array('%s') |
| 4089 | 4583 | ); |
| 4090 | 4584 | } |
| 4585 | + | |
| 4586 | + // The entry's restriction just changed — keep the OpenAI Vector Store | |
| 4587 | + // mirror consistent: non-public pulls the file (file_search has no | |
| 4588 | + // per-role filtering), public re-mirrors it (plan 15b5c6). | |
| 4589 | + if (class_exists('MxChat_Vectorstore_Manager')) { | |
| 4590 | + MxChat_Vectorstore_Manager::handle_role_change($source_url, 'default', $highest_role); | |
| 4591 | + } | |
| 4091 | 4592 | } |
| 4092 | 4593 | |
| 4093 | 4594 | public function mxchat_get_public_post_types() { |
| 4094 | 4595 | // Get all public post types |
| @@ -4159,64 +4660,63 @@ | ||
| 4159 | 4660 | |
| 4160 | 4661 | return $pinecone_data; |
| 4161 | 4662 | } |
| 4162 | 4663 | public function mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $vector_ids) { |
| 4163 | - //error_log('=== DEBUG: Starting mxchat_fetch_pinecone_vectors_by_ids ==='); | |
| 4164 | - | |
| 4165 | 4664 | $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? ''; |
| 4166 | 4665 | $host = $pinecone_options['mxchat_pinecone_host'] ?? ''; |
| 4666 | + $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? ''; | |
| 4167 | 4667 | |
| 4168 | 4668 | if (empty($api_key) || empty($host) || empty($vector_ids)) { |
| 4169 | - //error_log('DEBUG: Missing parameters for fetch by IDs'); | |
| 4170 | 4669 | return array(); |
| 4171 | 4670 | } |
| 4172 | 4671 | |
| 4173 | 4672 | try { |
| 4174 | - $fetch_url = "https://{$host}/vectors/fetch"; | |
| 4175 | - //error_log('DEBUG: Fetch URL: ' . $fetch_url); | |
| 4176 | - //error_log('DEBUG: Fetching ' . count($vector_ids) . ' vector IDs'); | |
| 4673 | + // NOTE (plan 793b82): /vectors/fetch is a GET endpoint with the ids | |
| 4674 | + // repeated in the query string (ids=a&ids=b — http_build_query would | |
| 4675 | + // emit ids[0]=a); the old POST here was answered 200-with-an-empty-body, | |
| 4676 | + // which read as "nothing indexed". Chunked at 100 ids to stay well | |
| 4677 | + // under the measured HTTP 414 URL-length boundary. | |
| 4678 | + $vectors = array(); | |
| 4679 | + foreach (array_chunk(array_values($vector_ids), 100) as $chunk) { | |
| 4680 | + $fetch_query = array(); | |
| 4681 | + foreach ($chunk as $fetch_vid) { | |
| 4682 | + $fetch_query[] = 'ids=' . rawurlencode($fetch_vid); | |
| 4683 | + } | |
| 4684 | + if (!empty($namespace)) { | |
| 4685 | + $fetch_query[] = 'namespace=' . rawurlencode($namespace); | |
| 4686 | + } | |
| 4177 | 4687 | |
| 4178 | - // Pinecone fetch API allows fetching specific vectors by ID | |
| 4179 | - $fetch_data = array( | |
| 4180 | - 'ids' => array_values($vector_ids) | |
| 4181 | - ); | |
| 4688 | + $response = wp_remote_get("https://{$host}/vectors/fetch?" . implode('&', $fetch_query), array( | |
| 4689 | + 'headers' => array( | |
| 4690 | + 'Api-Key' => $api_key, | |
| 4691 | + 'accept' => 'application/json' | |
| 4692 | + ), | |
| 4693 | + 'timeout' => 30 | |
| 4694 | + )); | |
| 4182 | 4695 | |
| 4183 | - $response = wp_remote_post($fetch_url, array( | |
| 4184 | - 'headers' => array( | |
| 4185 | - 'Api-Key' => $api_key, | |
| 4186 | - 'Content-Type' => 'application/json' | |
| 4187 | - ), | |
| 4188 | - 'body' => json_encode($fetch_data), | |
| 4189 | - 'timeout' => 30 | |
| 4190 | - )); | |
| 4696 | + if (is_wp_error($response)) { | |
| 4697 | + error_log('MxChat Pinecone: mxchat_fetch_pinecone_vectors_by_ids GET failed: ' . $response->get_error_message()); | |
| 4698 | + continue; | |
| 4699 | + } | |
| 4191 | 4700 | |
| 4192 | - if (is_wp_error($response)) { | |
| 4193 | - //error_log('DEBUG: Fetch by IDs WP error: ' . $response->get_error_message()); | |
| 4194 | - return array(); | |
| 4195 | - } | |
| 4701 | + if (wp_remote_retrieve_response_code($response) !== 200) { | |
| 4702 | + error_log('MxChat Pinecone: mxchat_fetch_pinecone_vectors_by_ids GET returned HTTP ' . wp_remote_retrieve_response_code($response)); | |
| 4703 | + continue; | |
| 4704 | + } | |
| 4196 | 4705 | |
| 4197 | - $response_code = wp_remote_retrieve_response_code($response); | |
| 4198 | - //error_log('DEBUG: Fetch response code: ' . $response_code); | |
| 4199 | - | |
| 4200 | - if ($response_code !== 200) { | |
| 4201 | - $error_body = wp_remote_retrieve_body($response); | |
| 4202 | - //error_log('DEBUG: Fetch failed with body: ' . $error_body); | |
| 4203 | - return array(); | |
| 4706 | + $data = json_decode(wp_remote_retrieve_body($response), true); | |
| 4707 | + if (isset($data['vectors']) && is_array($data['vectors'])) { | |
| 4708 | + $vectors += $data['vectors']; | |
| 4709 | + } | |
| 4204 | 4710 | } |
| 4205 | 4711 | |
| 4206 | - $body = wp_remote_retrieve_body($response); | |
| 4207 | - $data = json_decode($body, true); | |
| 4208 | - | |
| 4209 | - //error_log('DEBUG: Fetch response structure: ' . print_r(array_keys($data), true)); | |
| 4210 | - | |
| 4211 | - if (!isset($data['vectors'])) { | |
| 4212 | - //error_log('DEBUG: No vectors key in response'); | |
| 4712 | + if (empty($vectors)) { | |
| 4213 | 4713 | return array(); |
| 4214 | 4714 | } |
| 4215 | 4715 | |
| 4216 | 4716 | $processed_data = array(); |
| 4217 | 4717 | |
| 4218 | - foreach ($data['vectors'] as $vector_id => $vector_data) { | |
| 4718 | + foreach ($vectors as $vector_id => $vector_data) { | |
| 4219 | 4719 | $metadata = $vector_data['metadata'] ?? array(); |
| 4220 | 4720 | $source_url = $metadata['source_url'] ?? ''; |
| 4221 | 4721 | |
| 4222 | 4722 | if (!empty($source_url)) { |
| @@ -4287,8 +4787,11 @@ | ||
| 4287 | 4787 | */ |
| 4288 | 4788 | public function mxchat_scan_pinecone_for_processed_content($pinecone_options) { |
| 4289 | 4789 | $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? ''; |
| 4290 | 4790 | $host = $pinecone_options['mxchat_pinecone_host'] ?? ''; |
| 4791 | + // plan 793b82: this scan was namespace-blind — on a namespaced setup it | |
| 4792 | + // surveyed the default namespace and reported the wrong content as indexed. | |
| 4793 | + $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? ''; | |
| 4291 | 4794 | |
| 4292 | 4795 | if (empty($api_key) || empty($host)) { |
| 4293 | 4796 | return array(); |
| 4294 | 4797 | } |
| @@ -4323,8 +4826,12 @@ | ||
| 4323 | 4826 | 'topK' => 10000, |
| 4324 | 4827 | 'vector' => $random_vector |
| 4325 | 4828 | ); |
| 4326 | 4829 | |
| 4830 | + if (!empty($namespace)) { | |
| 4831 | + $query_data['namespace'] = $namespace; | |
| 4832 | + } | |
| 4833 | + | |
| 4327 | 4834 | $response = wp_remote_post($query_url, array( |
| 4328 | 4835 | 'headers' => array( |
| 4329 | 4836 | 'Api-Key' => $api_key, |
| 4330 | 4837 | 'Content-Type' => 'application/json' |
| @@ -4337,9 +4844,9 @@ | ||
| 4337 | 4844 | continue; |
| 4338 | 4845 | } |
| 4339 | 4846 | |
| 4340 | 4847 | $response_code = wp_remote_retrieve_response_code($response); |
| 4341 | - | |
| 4848 | + | |
| 4342 | 4849 | if ($response_code !== 200) { |
| 4343 | 4850 | continue; |
| 4344 | 4851 | } |
| 4345 | 4852 | |
| @@ -4548,13 +5055,23 @@ | ||
| 4548 | 5055 | $error_message = $error_json['error']['message'] ?? 'No message'; |
| 4549 | 5056 | //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type); |
| 4550 | 5057 | //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message); |
| 4551 | 5058 | |
| 4552 | - // Customize error message for common API errors | |
| 4553 | - if ($error_type === 'invalid_request_error' && strpos($error_message, 'API key') !== false) { | |
| 4554 | - $error_message = sprintf('Invalid %s API key for bot %s. Please check your API key in the bot settings.', $provider_name, $bot_id); | |
| 4555 | - } elseif ($error_type === 'authentication_error') { | |
| 4556 | - $error_message = sprintf('%s authentication failed for bot %s. Please verify your API key in the bot settings.', $provider_name, $bot_id); | |
| 5059 | + // Keep the provider's own diagnostic — a restricted-key 401 names the | |
| 5060 | + // exact missing scope, and replacing it with "check your API key" sent | |
| 5061 | + // a customer to regenerate two keys (plan 46b596). Same shape as | |
| 5062 | + // MxChat_Utils::embedding_failure_error() so both ingestion paths read | |
| 5063 | + // identically. Key never appears in provider messages, but scrub anyway. | |
| 5064 | + if ($error_type === 'invalid_request_error' || $error_type === 'authentication_error') { | |
| 5065 | + if (is_string($api_key) && $api_key !== '') { | |
| 5066 | + $error_message = str_replace($api_key, '[redacted]', $error_message); | |
| 5067 | + } | |
| 5068 | + $error_message = sprintf( | |
| 5069 | + 'Embedding failed (%s, HTTP %d): %s', | |
| 5070 | + $selected_model, | |
| 5071 | + $http_code, | |
| 5072 | + substr($error_message, 0, 300) | |
| 5073 | + ); | |
| 4557 | 5074 | } |
| 4558 | 5075 | |
| 4559 | 5076 | //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message); |
| 4560 | 5077 | return $error_message; |
| @@ -5066,25 +5583,38 @@ | ||
| 5066 | 5583 | /** |
| 5067 | 5584 | * Get all ACF fields for a specific post, excluding any fields the user has disabled |
| 5068 | 5585 | */ |
| 5069 | 5586 | public function mxchat_get_acf_fields_for_post($post_id) { |
| 5070 | - if (!function_exists('get_fields')) { | |
| 5587 | + if (!function_exists('get_field_objects')) { | |
| 5071 | 5588 | return array(); |
| 5072 | 5589 | } |
| 5073 | 5590 | |
| 5074 | - $fields = get_fields($post_id); | |
| 5075 | - if (!$fields || !is_array($fields)) { | |
| 5591 | + // Field OBJECTS, not get_fields(): exclusion matches on the field KEY | |
| 5592 | + // (unique per field) rather than the name (shared across groups — plan | |
| 5593 | + // 30e81f). ACF's own get_fields() is implemented as get_field_objects() | |
| 5594 | + // reduced to name => value, so the un-excluded reduction below is the | |
| 5595 | + // identical shape and order the previous get_fields() call produced. | |
| 5596 | + $field_objects = get_field_objects($post_id); | |
| 5597 | + if (!$field_objects || !is_array($field_objects)) { | |
| 5076 | 5598 | return array(); |
| 5077 | 5599 | } |
| 5078 | 5600 | |
| 5079 | - // Get excluded fields from settings | |
| 5080 | 5601 | $excluded_fields = get_option('mxchat_acf_excluded_fields', array()); |
| 5081 | - if (!empty($excluded_fields) && is_array($excluded_fields)) { | |
| 5082 | - foreach ($excluded_fields as $excluded_field) { | |
| 5083 | - if (isset($fields[$excluded_field])) { | |
| 5084 | - unset($fields[$excluded_field]); | |
| 5602 | + if (!is_array($excluded_fields)) { | |
| 5603 | + $excluded_fields = array(); | |
| 5604 | + } | |
| 5605 | + | |
| 5606 | + $fields = array(); | |
| 5607 | + foreach ($field_objects as $field_name => $field_object) { | |
| 5608 | + if (!empty($excluded_fields)) { | |
| 5609 | + $field_key = isset($field_object['key']) ? $field_object['key'] : ''; | |
| 5610 | + // Legacy name entries stay honored: a stored name whose group was | |
| 5611 | + // inactive at migration time still excludes every field wearing it. | |
| 5612 | + if (in_array($field_key, $excluded_fields, true) || in_array($field_name, $excluded_fields, true)) { | |
| 5613 | + continue; | |
| 5085 | 5614 | } |
| 5086 | 5615 | } |
| 5616 | + $fields[$field_name] = isset($field_object['value']) ? $field_object['value'] : null; | |
| 5087 | 5617 | } |
| 5088 | 5618 | |
| 5089 | 5619 | return $fields; |
| 5090 | 5620 | } |
| @@ -5096,8 +5626,11 @@ | ||
| 5096 | 5626 | if (!function_exists('acf_get_field_groups') || !function_exists('acf_get_fields')) { |
| 5097 | 5627 | return array(); |
| 5098 | 5628 | } |
| 5099 | 5629 | |
| 5630 | + // Keyed by GROUP KEY, not title (titles are not unique), and each field | |
| 5631 | + // entry carries its ACF field key — the unique identifier every toggle, | |
| 5632 | + // save, and index-time exclusion now runs on (plans 30e81f / bf57e0). | |
| 5100 | 5633 | $all_fields = array(); |
| 5101 | 5634 | $field_groups = acf_get_field_groups(); |
| 5102 | 5635 | |
| 5103 | 5636 | if (!empty($field_groups)) { |
| @@ -5103,16 +5636,21 @@ | ||
| 5103 | 5636 | if (!empty($field_groups)) { |
| 5104 | 5637 | foreach ($field_groups as $group) { |
| 5105 | 5638 | $group_fields = acf_get_fields($group['key']); |
| 5106 | 5639 | if (!empty($group_fields)) { |
| 5107 | - $all_fields[$group['title']] = array(); | |
| 5640 | + $entry = array( | |
| 5641 | + 'title' => $group['title'], | |
| 5642 | + 'fields' => array(), | |
| 5643 | + ); | |
| 5108 | 5644 | foreach ($group_fields as $field) { |
| 5109 | - $all_fields[$group['title']][] = array( | |
| 5110 | - 'name' => $field['name'], | |
| 5645 | + $entry['fields'][] = array( | |
| 5646 | + 'key' => $field['key'], | |
| 5647 | + 'name' => $field['name'], | |
| 5111 | 5648 | 'label' => $field['label'], |
| 5112 | - 'type' => $field['type'] | |
| 5649 | + 'type' => $field['type'] | |
| 5113 | 5650 | ); |
| 5114 | 5651 | } |
| 5652 | + $all_fields[$group['key']] = $entry; | |
| 5115 | 5653 | } |
| 5116 | 5654 | } |
| 5117 | 5655 | } |
| 5118 | 5656 | |
| @@ -5507,9 +6045,11 @@ | ||
| 5507 | 6045 | $parser = new \Smalot\PdfParser\Parser(); |
| 5508 | 6046 | $pdf = $parser->parseFile($pdf_path); |
| 5509 | 6047 | $pages = $pdf->getPages(); |
| 5510 | 6048 | $page_texts = array(); |
| 6049 | + $acf_page_num = 0; | |
| 5511 | 6050 | foreach ($pages as $page) { |
| 6051 | + $acf_page_num++; | |
| 5512 | 6052 | $page_text = ''; |
| 5513 | 6053 | try { |
| 5514 | 6054 | $page_text = $page->getText(); |
| 5515 | 6055 | } catch (\Exception $e) { |
| @@ -5515,8 +6055,9 @@ | ||
| 5515 | 6055 | } catch (\Exception $e) { |
| 5516 | 6056 | $page_text = ''; |
| 5517 | 6057 | } |
| 5518 | 6058 | if (!empty($page_text)) { |
| 6059 | + $page_text = MxChat_Utils::normalize_pdf_rtl($page_text, 'acf_pdf attachment ' . $attachment_id . ' page ' . $acf_page_num); | |
| 5519 | 6060 | $page_texts[] = $page_text; |
| 5520 | 6061 | } |
| 5521 | 6062 | } |
| 5522 | 6063 | $text = trim(implode("\n\n", $page_texts)); |
| @@ -5614,8 +6155,13 @@ | ||
| 5614 | 6155 | $this->mxchat_handle_post_update($post_id, $post, true); |
| 5615 | 6156 | } |
| 5616 | 6157 | |
| 5617 | 6158 | public function mxchat_handle_post_update($post_id, $post, $update) { |
| 6159 | + // The in-flight-update marker has done its job the moment post_updated runs; drop it | |
| 6160 | + // before any early return so it can never outlive its own save (a failed $wpdb->update | |
| 6161 | + // inside wp_insert_post returns after pre_post_update but before the transition). | |
| 6162 | + unset($this->pending_post_update[$post_id]); | |
| 6163 | + | |
| 5618 | 6164 | // Basic validation checks |
| 5619 | 6165 | if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) { |
| 5620 | 6166 | return; |
| 5621 | 6167 | } |
| @@ -5653,9 +6199,11 @@ | ||
| 5653 | 6199 | if ($previous_status === 'publish' && $post->post_status !== 'publish') { |
| 5654 | 6200 | // Use the stored URL from when it was published, or fall back to current permalink |
| 5655 | 6201 | $source_url = $previous_url ?: get_permalink($post_id); |
| 5656 | 6202 | |
| 5657 | - if ($source_url) { | |
| 6203 | + // mxchat_handle_status_transition already deleted for this post earlier in this | |
| 6204 | + // request (it fires first inside wp_insert_post); skip the redundant round-trip. | |
| 6205 | + if ($source_url && empty($this->transition_deleted_posts[$post_id])) { | |
| 5658 | 6206 | // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks) |
| 5659 | 6207 | MxChat_Utils::delete_chunks_for_url($source_url, 'default'); |
| 5660 | 6208 | } |
| 5661 | 6209 | |
| @@ -5684,209 +6232,402 @@ | ||
| 5684 | 6232 | set_transient($previous_url_key, $current_url, DAY_IN_SECONDS); |
| 5685 | 6233 | } |
| 5686 | 6234 | } |
| 5687 | 6235 | |
| 5688 | - // Only process currently published content for adding/updating | |
| 6236 | + // Only process currently published content for adding/updating. | |
| 6237 | + // transition_indexed_posts: mxchat_handle_status_transition's arrival edge may have | |
| 6238 | + // already indexed this post earlier in this request (editor publishes fire | |
| 6239 | + // transition_post_status first, then post_updated) — skip the duplicate embed. | |
| 6240 | + // Consume-once: the flag is cleared when honoured, so a LATER save of the same | |
| 6241 | + // post in one long-running process (WP-CLI scripts, importers) re-indexes normally. | |
| 5689 | 6242 | if ($post->post_status === 'publish') { |
| 6243 | + if (!empty($this->transition_indexed_posts[$post_id])) { | |
| 6244 | + unset($this->transition_indexed_posts[$post_id]); | |
| 6245 | + } else { | |
| 6246 | + $this->mxchat_index_published_post($post_id, $post); | |
| 6247 | + } | |
| 6248 | + } | |
| 6249 | + | |
| 6250 | + // Clean up the stored previous status if not used above | |
| 6251 | + if ($previous_status !== 'publish' || $post->post_status === 'publish') { | |
| 6252 | + delete_transient($previous_status_key); | |
| 6253 | + delete_transient($previous_url_key); | |
| 6254 | + } | |
| 6255 | +} | |
| 6256 | + | |
| 6257 | +/** | |
| 6258 | + * Index a published post into the knowledge base: preprocessing filter, content | |
| 6259 | + * assembly (title/excerpt/body), WooCommerce product enrichment, job_listing meta, | |
| 6260 | + * ACF fields (+ optional PDF extraction), whitelisted custom meta, embedding and | |
| 6261 | + * upsert, then tag-based role restriction. | |
| 6262 | + * | |
| 6263 | + * Shared by the post_updated auto-sync path (mxchat_handle_post_update) and the | |
| 6264 | + * transition_post_status arrival edge (mxchat_handle_status_transition), so | |
| 6265 | + * scheduled publishes (wp_publish_post) and direct status=publish inserts index | |
| 6266 | + * identically to editor saves (plan 3055e1). Pure extraction of the former | |
| 6267 | + * publish branch — body indentation retained to keep the diff reviewable. | |
| 6268 | + */ | |
| 6269 | +private function mxchat_index_published_post($post_id, $post) { | |
| 6270 | + $post_type = $post->post_type; | |
| 6271 | + | |
| 6272 | + // WooCommerce products are owned by the WC-object assembler (plan a3d60c): | |
| 6273 | + // whenever WooCommerce is active AND the integration is enabled, every | |
| 6274 | + // product save also fires save_post_product, which queues | |
| 6275 | + // mxchat_store_product_embedding on shutdown — and that writer runs LAST, | |
| 6276 | + // overwriting the same md5(permalink) row this path would write. Assembling | |
| 6277 | + // and embedding the product here was pure duplicate spend (measured: two | |
| 6278 | + // embedding calls per product save, second one wins). Skip ONLY under the | |
| 6279 | + // exact conditions the shutdown writer runs — same option read as its own | |
| 6280 | + // gate — because with the integration off (or WooCommerce inactive) this | |
| 6281 | + // path is the sole product indexer and must keep working. | |
| 6282 | + if ($post_type === 'product' | |
| 6283 | + && class_exists('WooCommerce') | |
| 6284 | + && isset($this->options['enable_woocommerce_integration']) | |
| 6285 | + && in_array($this->options['enable_woocommerce_integration'], ['1', 'on'])) { | |
| 6286 | + return; | |
| 6287 | + } | |
| 6288 | + | |
| 5690 | 6289 | // Get the source URL |
| 5691 | 6290 | $source_url = get_permalink($post_id); |
| 5692 | - | |
| 5693 | - // Get content with proper formatting (matching ajax_mxchat_process_selected_content) | |
| 5694 | - $title = get_the_title($post_id); | |
| 5695 | - $content = get_post_field('post_content', $post_id); | |
| 5696 | - $excerpt = get_post_field('post_excerpt', $post_id); | |
| 5697 | 6291 | |
| 5698 | - // Remove shortcode tags but preserve content inside them | |
| 5699 | - $content = $this->strip_shortcode_tags_preserve_content($content); | |
| 5700 | - $excerpt = $this->strip_shortcode_tags_preserve_content($excerpt); | |
| 6292 | + // A draft published programmatically (wp_publish_post) can reach this | |
| 6293 | + // point with an EMPTY post_name — wp_insert_post skips slug generation | |
| 6294 | + // for draft/pending — and get_permalink() then resolves to the bare | |
| 6295 | + // site root. A knowledge row keyed to the homepage cites the wrong URL | |
| 6296 | + // and answers homepage questions with this post's body, so refuse to | |
| 6297 | + // write it; the post indexes correctly on its next save, once the slug | |
| 6298 | + // exists. The empty-post_name test is what keeps a legitimate static | |
| 6299 | + // front page (which has a slug but a root permalink) indexable. | |
| 6300 | + // (Plan d138c4.) | |
| 6301 | + if ('' === $post->post_name | |
| 6302 | + && untrailingslashit($source_url) === untrailingslashit(home_url())) { | |
| 6303 | + return; | |
| 6304 | + } | |
| 5701 | 6305 | |
| 5702 | - // Strip tags but preserve structure (don't use 'the_content' filter as it may re-add shortcodes) | |
| 5703 | - $content = wp_strip_all_tags($content); | |
| 6306 | + /** | |
| 6307 | + * Allow developers to modify post data before processing into the knowledge base. | |
| 6308 | + * Same filter and signature as the manual bulk-import path | |
| 6309 | + * (ajax_mxchat_process_selected_content), so a callback registered once covers | |
| 6310 | + * every indexing route. Purely additive — zero behaviour change when unhooked. | |
| 6311 | + * Auto-sync runs under the 'default' bot context, matching the rest of this | |
| 6312 | + * function. | |
| 6313 | + * | |
| 6314 | + * @param WP_Post $post The post about to be indexed. | |
| 6315 | + * @param string $bot_id Bot context ('default' on auto-sync). | |
| 6316 | + */ | |
| 6317 | + $post = apply_filters('mxchat_before_process_post', $post, 'default'); | |
| 6318 | + if (!($post instanceof WP_Post)) { | |
| 6319 | + $post = get_post($post_id); // defend against a bad callback return | |
| 6320 | + } | |
| 5704 | 6321 | |
| 5705 | - // Combine title, short description (if exists), and content | |
| 5706 | - $final_content = $title . "\n\n"; | |
| 6322 | + // Assemble the indexable text via the shared post-kind assembler (a3d60c), | |
| 6323 | + // reading from the FILTERED post object — not re-fetched by ID, which would | |
| 6324 | + // discard it. Auto-sync reads content/excerpt in its historical | |
| 6325 | + // get_post_field() display context, never appended product custom tabs | |
| 6326 | + // (its product branch is reachable only with the WooCommerce integration | |
| 6327 | + // off), and gates ACF→PDF extraction behind its own opt-in option — | |
| 6328 | + // default OFF, because re-parsing every ACF PDF on every editor save is | |
| 6329 | + // expensive and most sites don't want it (the 25 MB size cap lives in the | |
| 6330 | + // shared extractor either way). | |
| 6331 | + $prepared = $this->mxchat_prepare_post_content_for_indexing($post_id, $post, array( | |
| 6332 | + 'read_display' => true, | |
| 6333 | + 'extract_acf_pdfs' => get_option('mxchat_auto_sync_acf_pdfs', '0') === '1', | |
| 6334 | + 'include_product_tabs' => false, | |
| 6335 | + )); | |
| 6336 | + $final_content = $prepared['content']; | |
| 5707 | 6337 | |
| 5708 | - // Add short description if it exists (WooCommerce products use post_excerpt for short description) | |
| 5709 | - if (!empty($excerpt)) { | |
| 5710 | - $final_content .= "Short Description: " . wp_strip_all_tags($excerpt) . "\n\n"; | |
| 6338 | + // Embedding decision — custom-provider-aware. Gating on a cloud API key | |
| 6339 | + // here silently killed auto-sync on keyless custom-embeddings sites, | |
| 6340 | + // because generate_embedding() routes custom FIRST and never needs the | |
| 6341 | + // key (plan cbd5fd). Silent-return shape preserved. | |
| 6342 | + $preflight = MxChat_Utils::embedding_preflight(get_option('mxchat_options')); | |
| 6343 | + if (!$preflight['ok']) { | |
| 6344 | + return; | |
| 5711 | 6345 | } |
| 6346 | + $api_key = $preflight['api_key']; | |
| 5712 | 6347 | |
| 5713 | - $final_content .= $content; | |
| 6348 | + // Use the centralized utility function for storage | |
| 6349 | + $result = MxChat_Utils::submit_content_to_db( | |
| 6350 | + $final_content, | |
| 6351 | + $source_url, | |
| 6352 | + $api_key, | |
| 6353 | + md5($source_url) // Vector ID for Pinecone | |
| 6354 | + ); | |
| 6355 | + | |
| 6356 | + // After successful storage, apply role restriction based on tags | |
| 6357 | + if (!is_wp_error($result)) { | |
| 6358 | + $this->apply_role_restriction_to_post($post_id, $source_url); | |
| 6359 | + } | |
| 6360 | +} | |
| 5714 | 6361 | |
| 5715 | - // For WooCommerce products, include pricing and product details | |
| 5716 | - if ($post_type === 'product' && class_exists('WooCommerce')) { | |
| 5717 | - $product = wc_get_product($post_id); | |
| 6362 | +/** | |
| 6363 | + * Shared post-fields content assembler (plan a3d60c) — the ONE body behind both | |
| 6364 | + * post-kind ingestion paths: manual bulk import (ajax_mxchat_process_selected_content) | |
| 6365 | + * and auto-sync (mxchat_index_published_post). Behavior-preserving extraction; the | |
| 6366 | + * measured per-caller differences ride $args instead of living as drifting copies: | |
| 6367 | + * | |
| 6368 | + * 'read_display' bool Auto-sync historically reads content/excerpt via | |
| 6369 | + * get_post_field() in its default 'display' context | |
| 6370 | + * (the post_content / post_excerpt display filters | |
| 6371 | + * fire); bulk import reads the raw properties. Inert | |
| 6372 | + * on a stock install — preserved per-path, not converged. | |
| 6373 | + * 'extract_acf_pdfs' bool Each caller passes its OWN option (bulk: | |
| 6374 | + * mxchat_acf_pdf_extraction; auto-sync: | |
| 6375 | + * mxchat_auto_sync_acf_pdfs) — the two-option design | |
| 6376 | + * is deliberate (plan 11720c). Gates BOTH the PDF-id | |
| 6377 | + * collection walk and the extraction loop; the ids are | |
| 6378 | + * only ever read inside the extraction branch, so | |
| 6379 | + * gating collection is output-identical on every install. | |
| 6380 | + * 'include_product_tabs' bool The bulk path has always appended yikes_woo custom | |
| 6381 | + * tabs to product content; the auto-sync product branch | |
| 6382 | + * (reachable only with the WooCommerce integration off) | |
| 6383 | + * never did. Preserved per-path — converging it would be | |
| 6384 | + * a behavior change, recorded on the plan instead. | |
| 6385 | + * | |
| 6386 | + * Returns array: 'content' (the assembled indexable text), 'acf_fields_found' and | |
| 6387 | + * 'pdf_extracted_count' (the bulk path reports both in its AJAX response). | |
| 6388 | + */ | |
| 6389 | +private function mxchat_prepare_post_content_for_indexing($post_id, $post, $args) { | |
| 6390 | + $read_display = !empty($args['read_display']); | |
| 6391 | + $extract_acf_pdfs = !empty($args['extract_acf_pdfs']); | |
| 6392 | + $include_product_tabs = !empty($args['include_product_tabs']); | |
| 5718 | 6393 | |
| 5719 | - if ($product) { | |
| 5720 | - // Get pricing information | |
| 5721 | - $regular_price = $product->get_regular_price(); | |
| 5722 | - $sale_price = $product->get_sale_price(); | |
| 5723 | - $price = $product->get_price(); | |
| 5724 | - $sku = $product->get_sku(); | |
| 6394 | + // Raw post_title, NOT get_the_title(): the_title applies wptexturize + | |
| 6395 | + // convert_chars and prepends the "Protected:" / "Private:" display chrome. | |
| 6396 | + // The knowledge base stores facts, not display strings. Entity decode at | |
| 6397 | + // output time (single-pass, shared helper) — a stored `&` embeds worse | |
| 6398 | + // than `&` and gets quoted back to visitors (d2c92e). | |
| 6399 | + $content = $this->mxchat_decode_entities_for_indexing($post->post_title) . "\n\n"; | |
| 5725 | 6400 | |
| 5726 | - // Get currency symbol | |
| 5727 | - $currency_symbol = get_woocommerce_currency_symbol(); | |
| 6401 | + $raw_excerpt = $read_display ? get_post_field('post_excerpt', $post) : $post->post_excerpt; | |
| 6402 | + $raw_content = $read_display ? get_post_field('post_content', $post) : $post->post_content; | |
| 5728 | 6403 | |
| 5729 | - // Add pricing information | |
| 5730 | - $final_content .= "\n"; | |
| 5731 | - if (!empty($regular_price)) { | |
| 5732 | - $final_content .= "Price: " . $currency_symbol . $regular_price . "\n"; | |
| 5733 | - } elseif (!empty($price)) { | |
| 5734 | - $final_content .= "Price: " . $currency_symbol . $price . "\n"; | |
| 5735 | - } | |
| 6404 | + // Add short description if it exists (WooCommerce products use post_excerpt for short description) | |
| 6405 | + // Strip FIRST, then test: an excerpt that is nothing but shortcodes strips to | |
| 6406 | + // empty, and testing the raw value emitted a bare "Short Description: " label | |
| 6407 | + // with no value after it. trim() only in the TEST — the emitted value is | |
| 6408 | + // untouched, so a populated excerpt is byte-identical to before. A | |
| 6409 | + // whitespace-only excerpt is an empty excerpt and must not produce a labelled | |
| 6410 | + // line with nothing after it. | |
| 6411 | + $clean_excerpt = $this->strip_shortcode_tags_preserve_content($raw_excerpt); | |
| 6412 | + if (trim($clean_excerpt) !== '') { | |
| 6413 | + $content .= "Short Description: " . $this->mxchat_decode_entities_for_indexing(wp_strip_all_tags($clean_excerpt)) . "\n\n"; | |
| 6414 | + } | |
| 5736 | 6415 | |
| 5737 | - if (!empty($sale_price) && $sale_price !== $regular_price) { | |
| 5738 | - $final_content .= "Sale Price: " . $currency_symbol . $sale_price . "\n"; | |
| 5739 | - } | |
| 6416 | + // Main content — remove shortcode tags but preserve content inside them, then | |
| 6417 | + // strip tags (don't use 'the_content' filter as it may re-add shortcodes). | |
| 6418 | + $clean_content = $this->strip_shortcode_tags_preserve_content($raw_content); | |
| 6419 | + $content .= $this->mxchat_decode_entities_for_indexing(wp_strip_all_tags($clean_content)); | |
| 5740 | 6420 | |
| 5741 | - // Handle variable products - show price range | |
| 5742 | - if ($product->is_type('variable')) { | |
| 5743 | - $min_price = $product->get_variation_price('min'); | |
| 5744 | - $max_price = $product->get_variation_price('max'); | |
| 5745 | - if ($min_price !== $max_price) { | |
| 5746 | - $final_content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n"; | |
| 5747 | - } | |
| 5748 | - } | |
| 6421 | + // WooCommerce product enrichment (post-fields kind). The WC-object assembler | |
| 6422 | + // (mxchat_prepare_product_content_for_indexing) owns product rows whenever the | |
| 6423 | + // integration is on; this branch serves the bulk import (all configurations) | |
| 6424 | + // and auto-sync with the integration off. | |
| 6425 | + if (get_post_type($post_id) === 'product' && class_exists('WooCommerce')) { | |
| 6426 | + $product = wc_get_product($post_id); | |
| 5749 | 6427 | |
| 5750 | - if (!empty($sku)) { | |
| 5751 | - $final_content .= "SKU: " . $sku . "\n"; | |
| 5752 | - } | |
| 6428 | + if ($product) { | |
| 6429 | + $content .= "\n"; | |
| 6430 | + $content .= $this->mxchat_woo_product_summary_lines($product); | |
| 6431 | + } | |
| 5753 | 6432 | |
| 5754 | - // Get product categories | |
| 5755 | - $categories = wp_get_post_terms($post_id, 'product_cat', array('fields' => 'names')); | |
| 5756 | - if (!empty($categories) && !is_wp_error($categories)) { | |
| 5757 | - $final_content .= "Categories: " . implode(', ', $categories) . "\n"; | |
| 5758 | - } | |
| 5759 | - } | |
| 6433 | + if ($include_product_tabs) { | |
| 6434 | + $content .= $this->mxchat_woo_custom_tabs_text($post_id); | |
| 5760 | 6435 | } |
| 6436 | + } | |
| 5761 | 6437 | |
| 5762 | - // For custom post types like job_listing, include additional fields | |
| 5763 | - if ($post_type === 'job_listing') { | |
| 5764 | - // Add job-specific meta if available | |
| 5765 | - $job_location = get_post_meta($post_id, '_job_location', true); | |
| 5766 | - if (!empty($job_location)) { | |
| 5767 | - $final_content .= "\n\nLocation: " . $job_location; | |
| 5768 | - } | |
| 6438 | + // For custom post types like job_listing, include additional fields | |
| 6439 | + if (get_post_type($post_id) === 'job_listing') { | |
| 6440 | + // Add job-specific meta if available | |
| 6441 | + $job_location = get_post_meta($post_id, '_job_location', true); | |
| 6442 | + if (!empty($job_location)) { | |
| 6443 | + $content .= "\n\nLocation: " . $job_location; | |
| 6444 | + } | |
| 5769 | 6445 | |
| 5770 | - // Get job type terms | |
| 5771 | - $job_types = get_the_terms($post_id, 'job_listing_type'); | |
| 5772 | - if (!empty($job_types) && !is_wp_error($job_types)) { | |
| 5773 | - $types = array(); | |
| 5774 | - foreach ($job_types as $type) { | |
| 5775 | - $types[] = $type->name; | |
| 5776 | - } | |
| 5777 | - $final_content .= "\n\nJob Type: " . implode(', ', $types); | |
| 6446 | + // Get job type terms | |
| 6447 | + $job_types = get_the_terms($post_id, 'job_listing_type'); | |
| 6448 | + if (!empty($job_types) && !is_wp_error($job_types)) { | |
| 6449 | + $types = array(); | |
| 6450 | + foreach ($job_types as $type) { | |
| 6451 | + $types[] = $type->name; | |
| 5778 | 6452 | } |
| 6453 | + $content .= "\n\nJob Type: " . implode(', ', $types); | |
| 6454 | + } | |
| 5779 | 6455 | |
| 5780 | - // Get company name if available | |
| 5781 | - $company_name = get_post_meta($post_id, '_company_name', true); | |
| 5782 | - if (!empty($company_name)) { | |
| 5783 | - $final_content .= "\n\nCompany: " . $company_name; | |
| 5784 | - } | |
| 6456 | + // Get company name if available | |
| 6457 | + $company_name = get_post_meta($post_id, '_company_name', true); | |
| 6458 | + if (!empty($company_name)) { | |
| 6459 | + $content .= "\n\nCompany: " . $company_name; | |
| 5785 | 6460 | } |
| 6461 | + } | |
| 5786 | 6462 | |
| 5787 | - // ADD ACF FIELDS SUPPORT (matches ajax_mxchat_process_selected_content behavior) | |
| 5788 | - $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id); | |
| 5789 | - if (!empty($acf_fields)) { | |
| 5790 | - $acf_content_parts = array(); | |
| 5791 | - $pdf_attachment_ids = array(); | |
| 6463 | + // ADD ACF FIELDS SUPPORT | |
| 6464 | + $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id); | |
| 6465 | + $pdf_extracted_count = 0; | |
| 6466 | + if (!empty($acf_fields)) { | |
| 6467 | + $acf_content_parts = array(); | |
| 6468 | + $pdf_attachment_ids = array(); | |
| 5792 | 6469 | |
| 5793 | - foreach ($acf_fields as $field_name => $field_value) { | |
| 5794 | - $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id); | |
| 5795 | - if (!empty($formatted_value)) { | |
| 5796 | - // Convert field name to readable label | |
| 5797 | - $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name)); | |
| 5798 | - $acf_content_parts[] = $field_label . ": " . $formatted_value; | |
| 5799 | - } | |
| 6470 | + foreach ($acf_fields as $field_name => $field_value) { | |
| 6471 | + $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id); | |
| 5800 | 6472 | |
| 6473 | + if (!empty($formatted_value)) { | |
| 6474 | + // Both separators: a hyphenated ACF name should read as words. | |
| 6475 | + $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name)); | |
| 6476 | + $acf_content_parts[] = $field_label . ": " . $formatted_value; | |
| 6477 | + } | |
| 6478 | + | |
| 6479 | + // Walk this field's value tree for any PDF attachment references and | |
| 6480 | + // queue them for extraction — only when this caller's PDF option is on. | |
| 6481 | + if ($extract_acf_pdfs) { | |
| 5801 | 6482 | $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids); |
| 5802 | 6483 | } |
| 6484 | + } | |
| 5803 | 6485 | |
| 5804 | - if (!empty($acf_content_parts)) { | |
| 5805 | - $final_content .= "\n\n" . implode("\n", $acf_content_parts); | |
| 5806 | - } | |
| 6486 | + if (!empty($acf_content_parts)) { | |
| 6487 | + $content .= "\n\n" . implode("\n", $acf_content_parts); | |
| 6488 | + } | |
| 5807 | 6489 | |
| 5808 | - // Gate the auto-sync PDF-extraction loop behind an opt-in option. | |
| 5809 | - // Mirrors the per-batch checkbox the manual content selector has; the | |
| 5810 | - // 25 MB size cap lives in the shared extractor so it applies in both | |
| 5811 | - // paths regardless. Default OFF — re-parsing every ACF PDF on every | |
| 5812 | - // editor save is expensive and most sites don't want it. | |
| 5813 | - $autosync_extract_acf_pdfs = get_option('mxchat_auto_sync_acf_pdfs', '0') === '1'; | |
| 5814 | - if ($autosync_extract_acf_pdfs && !empty($pdf_attachment_ids)) { | |
| 5815 | - $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids))); | |
| 5816 | - $pdf_sections = array(); | |
| 5817 | - foreach ($pdf_attachment_ids as $att_id) { | |
| 5818 | - $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id); | |
| 5819 | - if (!empty($pdf_text)) { | |
| 5820 | - $pdf_title = get_the_title($att_id); | |
| 5821 | - $pdf_url = wp_get_attachment_url($att_id); | |
| 5822 | - $header = 'PDF Attachment'; | |
| 5823 | - if (!empty($pdf_title)) { | |
| 5824 | - $header .= ': ' . $pdf_title; | |
| 5825 | - } | |
| 5826 | - if (!empty($pdf_url)) { | |
| 5827 | - $header .= ' (' . $pdf_url . ')'; | |
| 5828 | - } | |
| 5829 | - $pdf_sections[] = $header . "\n" . $pdf_text; | |
| 6490 | + // Extract text from each unique PDF found in ACF fields and append as a labeled section | |
| 6491 | + if ($extract_acf_pdfs && !empty($pdf_attachment_ids)) { | |
| 6492 | + $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids))); | |
| 6493 | + $pdf_sections = array(); | |
| 6494 | + foreach ($pdf_attachment_ids as $att_id) { | |
| 6495 | + $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id); | |
| 6496 | + if (!empty($pdf_text)) { | |
| 6497 | + $pdf_title = get_the_title($att_id); | |
| 6498 | + $pdf_url = wp_get_attachment_url($att_id); | |
| 6499 | + $header = 'PDF Attachment'; | |
| 6500 | + if (!empty($pdf_title)) { | |
| 6501 | + $header .= ': ' . $pdf_title; | |
| 5830 | 6502 | } |
| 6503 | + if (!empty($pdf_url)) { | |
| 6504 | + $header .= ' (' . $pdf_url . ')'; | |
| 6505 | + } | |
| 6506 | + $pdf_sections[] = $header . "\n" . $pdf_text; | |
| 6507 | + $pdf_extracted_count++; | |
| 5831 | 6508 | } |
| 5832 | - if (!empty($pdf_sections)) { | |
| 5833 | - $final_content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections); | |
| 5834 | - } | |
| 5835 | 6509 | } |
| 6510 | + if (!empty($pdf_sections)) { | |
| 6511 | + $content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections); | |
| 6512 | + } | |
| 5836 | 6513 | } |
| 6514 | + } | |
| 5837 | 6515 | |
| 5838 | - // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields) | |
| 5839 | - $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id); | |
| 5840 | - if (!empty($custom_meta)) { | |
| 5841 | - $meta_content_parts = array(); | |
| 6516 | + // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields) | |
| 6517 | + $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id); | |
| 6518 | + if (!empty($custom_meta)) { | |
| 6519 | + $meta_content_parts = array(); | |
| 5842 | 6520 | |
| 5843 | - foreach ($custom_meta as $meta_key => $meta_value) { | |
| 5844 | - // Convert meta key to readable label | |
| 5845 | - $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key)); | |
| 5846 | - $meta_content_parts[] = $meta_label . ": " . $meta_value; | |
| 5847 | - } | |
| 6521 | + foreach ($custom_meta as $meta_key => $meta_value) { | |
| 6522 | + // Convert meta key to readable label | |
| 6523 | + $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key)); | |
| 6524 | + $meta_content_parts[] = $meta_label . ": " . $meta_value; | |
| 6525 | + } | |
| 5848 | 6526 | |
| 5849 | - if (!empty($meta_content_parts)) { | |
| 5850 | - $final_content .= "\n\n" . implode("\n", $meta_content_parts); | |
| 6527 | + if (!empty($meta_content_parts)) { | |
| 6528 | + $content .= "\n\n" . implode("\n", $meta_content_parts); | |
| 6529 | + } | |
| 6530 | + } | |
| 6531 | + | |
| 6532 | + return array( | |
| 6533 | + 'content' => $content, | |
| 6534 | + 'acf_fields_found' => count($acf_fields), | |
| 6535 | + 'pdf_extracted_count' => $pdf_extracted_count, | |
| 6536 | + ); | |
| 6537 | +} | |
| 6538 | + | |
| 6539 | +/** | |
| 6540 | + * Shared WC-object product assembler (plan a3d60c) — the ONE body behind the two | |
| 6541 | + * WooCommerce-object ingestion paths: the auto-sync product writer | |
| 6542 | + * (mxchat_store_product_embedding) and the URL/sitemap product import | |
| 6543 | + * (mxchat_extract_woocommerce_product_content). Assembles from the WC_Product, | |
| 6544 | + * the authoritative source for product rows (scope decision on the plan). | |
| 6545 | + */ | |
| 6546 | +private function mxchat_prepare_product_content_for_indexing($product) { | |
| 6547 | + $title = $product->get_name(); | |
| 6548 | + $description = $product->get_description(); | |
| 6549 | + $short_description = $product->get_short_description(); | |
| 6550 | + | |
| 6551 | + // Format content consistently | |
| 6552 | + $content = $title . "\n\n"; | |
| 6553 | + | |
| 6554 | + if (!empty($short_description)) { | |
| 6555 | + $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n"; | |
| 6556 | + } | |
| 6557 | + | |
| 6558 | + if (!empty($description)) { | |
| 6559 | + $content .= wp_strip_all_tags($description) . "\n\n"; | |
| 6560 | + } | |
| 6561 | + | |
| 6562 | + $content .= $this->mxchat_woo_product_summary_lines($product); | |
| 6563 | + $content .= $this->mxchat_woo_custom_tabs_text($product->get_id()); | |
| 6564 | + | |
| 6565 | + return $content; | |
| 6566 | +} | |
| 6567 | + | |
| 6568 | +/** | |
| 6569 | + * Pricing + SKU + categories lines for a product — shared by both assembler kinds | |
| 6570 | + * (the post-fields product enrichment and the WC-object assembler). | |
| 6571 | + */ | |
| 6572 | +private function mxchat_woo_product_summary_lines($product) { | |
| 6573 | + // Add pricing information (base-currency pinned — see mxchat_product_price_lines) | |
| 6574 | + $lines = $this->mxchat_product_price_lines($product); | |
| 6575 | + | |
| 6576 | + $sku = $product->get_sku(); | |
| 6577 | + if (!empty($sku)) { | |
| 6578 | + $lines .= "SKU: " . $sku . "\n"; | |
| 6579 | + } | |
| 6580 | + | |
| 6581 | + // Get product categories | |
| 6582 | + $categories = wp_get_post_terms($product->get_id(), 'product_cat', array('fields' => 'names')); | |
| 6583 | + if (!empty($categories) && !is_wp_error($categories)) { | |
| 6584 | + $lines .= "Categories: " . implode(', ', $categories) . "\n"; | |
| 6585 | + } | |
| 6586 | + | |
| 6587 | + return $lines; | |
| 6588 | +} | |
| 6589 | + | |
| 6590 | +/** | |
| 6591 | + * Custom Product Tabs text (supports "Custom Product Tabs for WooCommerce" by | |
| 6592 | + * Code Parrots) — direct tabs plus applied reusable/saved tabs. The ONE copy of | |
| 6593 | + * the yikes_woo logic; three sites carried byte-identical clones before a3d60c. | |
| 6594 | + */ | |
| 6595 | +private function mxchat_woo_custom_tabs_text($product_id) { | |
| 6596 | + $text = ''; | |
| 6597 | + | |
| 6598 | + $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true); | |
| 6599 | + if (!empty($custom_tabs) && is_array($custom_tabs)) { | |
| 6600 | + foreach ($custom_tabs as $tab) { | |
| 6601 | + $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : ''); | |
| 6602 | + $tab_content = isset($tab['content']) ? $tab['content'] : ''; | |
| 6603 | + | |
| 6604 | + if (!empty($tab_title) && !empty($tab_content)) { | |
| 6605 | + $text .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n"; | |
| 5851 | 6606 | } |
| 5852 | 6607 | } |
| 6608 | + } | |
| 5853 | 6609 | |
| 5854 | - // Get API key with proper model detection | |
| 5855 | - $options = get_option('mxchat_options'); | |
| 5856 | - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002'; | |
| 5857 | - | |
| 5858 | - if (strpos($selected_model, 'voyage') === 0) { | |
| 5859 | - $api_key = $options['voyage_api_key'] ?? ''; | |
| 5860 | - } elseif (strpos($selected_model, 'gemini-embedding') === 0) { | |
| 5861 | - $api_key = $options['gemini_api_key'] ?? ''; | |
| 5862 | - } else { | |
| 5863 | - $api_key = $options['api_key'] ?? ''; | |
| 6610 | + // Also check for reusable/saved tabs applied to this product | |
| 6611 | + $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true); | |
| 6612 | + if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) { | |
| 6613 | + $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array()); | |
| 6614 | + if (!empty($saved_tabs) && is_array($saved_tabs)) { | |
| 6615 | + foreach ($applied_saved_tabs as $saved_tab_id) { | |
| 6616 | + if (isset($saved_tabs[$saved_tab_id])) { | |
| 6617 | + $tab = $saved_tabs[$saved_tab_id]; | |
| 6618 | + $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : ''); | |
| 6619 | + $tab_content = isset($tab['content']) ? $tab['content'] : ''; | |
| 6620 | + | |
| 6621 | + if (!empty($tab_title) && !empty($tab_content)) { | |
| 6622 | + $text .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n"; | |
| 6623 | + } | |
| 6624 | + } | |
| 6625 | + } | |
| 5864 | 6626 | } |
| 5865 | - | |
| 5866 | - if (empty($api_key)) { | |
| 5867 | - return; | |
| 5868 | - } | |
| 5869 | - | |
| 5870 | - // Use the centralized utility function for storage | |
| 5871 | - $result = MxChat_Utils::submit_content_to_db( | |
| 5872 | - $final_content, | |
| 5873 | - $source_url, | |
| 5874 | - $api_key, | |
| 5875 | - md5($source_url) // Vector ID for Pinecone | |
| 5876 | - ); | |
| 5877 | - | |
| 5878 | - // After successful storage, apply role restriction based on tags | |
| 5879 | - if (!is_wp_error($result)) { | |
| 5880 | - $this->apply_role_restriction_to_post($post_id, $source_url); | |
| 5881 | - } | |
| 5882 | 6627 | } |
| 5883 | - | |
| 5884 | - // Clean up the stored previous status if not used above | |
| 5885 | - if ($previous_status !== 'publish' || $post->post_status === 'publish') { | |
| 5886 | - delete_transient($previous_status_key); | |
| 5887 | - delete_transient($previous_url_key); | |
| 5888 | - } | |
| 6628 | + | |
| 6629 | + return $text; | |
| 5889 | 6630 | } |
| 5890 | 6631 | |
| 5891 | 6632 | /** |
| 5892 | 6633 | * Store the post status and URL before update to detect status transitions |
| @@ -5892,8 +6633,12 @@ | ||
| 5892 | 6633 | * Store the post status and URL before update to detect status transitions |
| 5893 | 6634 | * This runs before the post is actually updated in the database |
| 5894 | 6635 | */ |
| 5895 | 6636 | public function mxchat_store_pre_update_status($post_id, $data) { |
| 6637 | + // Core is inside wp_insert_post's update branch, so a post_updated WILL fire later | |
| 6638 | + // this request and can consume the arrival-edge guard (plan a664f3). | |
| 6639 | + $this->pending_post_update[$post_id] = true; | |
| 6640 | + | |
| 5896 | 6641 | // Get the current post from database (before update) |
| 5897 | 6642 | $current_post = get_post($post_id); |
| 5898 | 6643 | |
| 5899 | 6644 | if ($current_post) { |
| @@ -5909,8 +6654,407 @@ | ||
| 5909 | 6654 | } |
| 5910 | 6655 | } |
| 5911 | 6656 | } |
| 5912 | 6657 | |
| 6658 | +/** | |
| 6659 | + * Whether auto-sync is enabled for a post type (mirrors the checks used by the | |
| 6660 | + * update/delete handlers; kept as one helper so new call sites cannot drift). | |
| 6661 | + */ | |
| 6662 | +private function mxchat_is_auto_sync_enabled($post_type) { | |
| 6663 | + if ($post_type === 'post') { | |
| 6664 | + return get_option('mxchat_auto_sync_posts') === '1'; | |
| 6665 | + } | |
| 6666 | + if ($post_type === 'page') { | |
| 6667 | + return get_option('mxchat_auto_sync_pages') === '1'; | |
| 6668 | + } | |
| 6669 | + return get_option('mxchat_auto_sync_' . $post_type) === '1'; | |
| 6670 | +} | |
| 6671 | + | |
| 6672 | +/** | |
| 6673 | + * Remove a post's vectors the moment it leaves 'publish', using the authoritative | |
| 6674 | + * old status core passes to transition_post_status — no transient involved (plan 816fb1). | |
| 6675 | + * | |
| 6676 | + * Covers status changes that never route through wp_update_post (scheduled-expiry | |
| 6677 | + * plugins and others that flip post_status directly and call wp_transition_post_status), | |
| 6678 | + * where neither pre_post_update nor post_updated fires and the old detection missed. | |
| 6679 | + */ | |
| 6680 | +public function mxchat_handle_status_transition($new_status, $old_status, $post) { | |
| 6681 | + if (!($post instanceof WP_Post) || wp_is_post_revision($post->ID)) { | |
| 6682 | + return; | |
| 6683 | + } | |
| 6684 | + | |
| 6685 | + // Arrival edge (plan 3055e1): a post BECOMING published is indexed here, because | |
| 6686 | + // wp_publish_post() — the path scheduled posts take via check_and_publish_future_post — | |
| 6687 | + // and direct wp_insert_post(status=publish) creates never fire post_updated, so the | |
| 6688 | + // auto-sync ADD path alone misses them. Editor publishes also pass through here; | |
| 6689 | + // the transition_indexed_posts guard keeps mxchat_handle_post_update from embedding | |
| 6690 | + // a second time in the same request. | |
| 6691 | + if ($new_status === 'publish' && $old_status !== 'publish') { | |
| 6692 | + if ($this->mxchat_is_auto_sync_enabled($post->post_type)) { | |
| 6693 | + $this->mxchat_index_published_post($post->ID, $post); | |
| 6694 | + | |
| 6695 | + // Arm the double-fire guard ONLY when a post_updated is actually coming to | |
| 6696 | + // consume it (plan a664f3). Two publish paths never fire post_updated at all: | |
| 6697 | + // a direct wp_insert_post(status=publish) create, and wp_publish_post() — the | |
| 6698 | + // call check_and_publish_future_post() makes for scheduled posts. Arming the | |
| 6699 | + // guard unconditionally left it set with nothing to consume it, so the NEXT | |
| 6700 | + // update of that post was swallowed entirely: zero embed calls, no knowledge | |
| 6701 | + // -base row, silently. Consume-once on this side too, so a guard can never | |
| 6702 | + // outlive the single save it was armed for. | |
| 6703 | + if (!empty($this->pending_post_update[$post->ID])) { | |
| 6704 | + unset($this->pending_post_update[$post->ID]); | |
| 6705 | + $this->transition_indexed_posts[$post->ID] = true; | |
| 6706 | + } | |
| 6707 | + } | |
| 6708 | + return; | |
| 6709 | + } | |
| 6710 | + | |
| 6711 | + // Only the publish -> not-publish edge matters here. | |
| 6712 | + if ($old_status !== 'publish' || $new_status === 'publish') { | |
| 6713 | + return; | |
| 6714 | + } | |
| 6715 | + // Trash is handled by mxchat_handle_post_delete (wp_trash_post) with pre-trash URL | |
| 6716 | + // resolution; skip to avoid a second network round-trip per trash. | |
| 6717 | + if ($new_status === 'trash') { | |
| 6718 | + return; | |
| 6719 | + } | |
| 6720 | + if (!$this->mxchat_is_auto_sync_enabled($post->post_type)) { | |
| 6721 | + return; | |
| 6722 | + } | |
| 6723 | + | |
| 6724 | + $urls = array(); | |
| 6725 | + | |
| 6726 | + // The DB may already hold the new status when this fires, so get_permalink() on the | |
| 6727 | + // live post could build a draft-style URL whose md5 misses the stored vector IDs. | |
| 6728 | + // Reconstruct the published permalink from a clone instead. | |
| 6729 | + $published_clone = clone $post; | |
| 6730 | + $published_clone->post_status = 'publish'; | |
| 6731 | + $published_url = get_permalink($published_clone); | |
| 6732 | + if ($published_url) { | |
| 6733 | + $urls[] = $published_url; | |
| 6734 | + } | |
| 6735 | + | |
| 6736 | + // Honour the pre-update capture when present (covers a slug change in the same save). | |
| 6737 | + $previous_url = get_transient('mxchat_prev_url_' . $post->ID); | |
| 6738 | + if (!empty($previous_url)) { | |
| 6739 | + $urls[] = $previous_url; | |
| 6740 | + } | |
| 6741 | + | |
| 6742 | + foreach (array_unique($urls) as $url) { | |
| 6743 | + MxChat_Utils::delete_chunks_for_url($url, 'default'); | |
| 6744 | + } | |
| 6745 | + | |
| 6746 | + if (!empty($urls)) { | |
| 6747 | + $this->transition_deleted_posts[$post->ID] = true; | |
| 6748 | + } | |
| 6749 | +} | |
| 6750 | + | |
| 6751 | +/** | |
| 6752 | + * WP-CLI: remove knowledge-base entries left behind by posts that were unpublished, | |
| 6753 | + * trashed, or made private before the transition_post_status handler existed. | |
| 6754 | + * | |
| 6755 | + * Walks every auto-synced post type's non-published posts, reconstructs each one's | |
| 6756 | + * published-era permalink, and deletes its vectors (routes to Pinecone or the WP table). | |
| 6757 | + * Deletion is idempotent, so never-indexed posts are a cheap no-op. | |
| 6758 | + * | |
| 6759 | + * ## OPTIONS | |
| 6760 | + * | |
| 6761 | + * [--dry-run] | |
| 6762 | + * : Report what would be removed without deleting anything. | |
| 6763 | + * | |
| 6764 | + * ## EXAMPLES | |
| 6765 | + * | |
| 6766 | + * wp mxchat prune-unpublished --dry-run | |
| 6767 | + * wp mxchat prune-unpublished | |
| 6768 | + */ | |
| 6769 | +public function cli_prune_unpublished($args, $assoc_args) { | |
| 6770 | + global $wpdb; | |
| 6771 | + $dry_run = !empty($assoc_args['dry-run']); | |
| 6772 | + $table = $wpdb->prefix . 'mxchat_system_prompt_content'; | |
| 6773 | + | |
| 6774 | + $candidate_types = array_merge(array('post', 'page'), array_values(get_post_types(array('_builtin' => false), 'names'))); | |
| 6775 | + $synced_types = array(); | |
| 6776 | + foreach ($candidate_types as $type) { | |
| 6777 | + if ($this->mxchat_is_auto_sync_enabled($type)) { | |
| 6778 | + $synced_types[] = $type; | |
| 6779 | + } | |
| 6780 | + } | |
| 6781 | + if (empty($synced_types)) { | |
| 6782 | + WP_CLI::success('No post types have auto-sync enabled; nothing to prune.'); | |
| 6783 | + return; | |
| 6784 | + } | |
| 6785 | + | |
| 6786 | + $scanned = 0; | |
| 6787 | + $pruned = 0; | |
| 6788 | + $paged = 1; | |
| 6789 | + do { | |
| 6790 | + $query = new WP_Query(array( | |
| 6791 | + 'post_type' => $synced_types, | |
| 6792 | + 'post_status' => array('draft', 'pending', 'private', 'future', 'trash'), | |
| 6793 | + 'posts_per_page' => 100, | |
| 6794 | + 'paged' => $paged, | |
| 6795 | + 'fields' => 'ids', | |
| 6796 | + )); | |
| 6797 | + foreach ($query->posts as $post_id) { | |
| 6798 | + $post = get_post($post_id); | |
| 6799 | + if (!$post) { | |
| 6800 | + continue; | |
| 6801 | + } | |
| 6802 | + $scanned++; | |
| 6803 | + | |
| 6804 | + // Rebuild the permalink the post had while published: publish-status clone, | |
| 6805 | + // with wp_trash_post's __trashed slug suffix stripped for trashed posts. | |
| 6806 | + $clone = clone $post; | |
| 6807 | + $clone->post_status = 'publish'; | |
| 6808 | + if (substr($clone->post_name, -9) === '__trashed') { | |
| 6809 | + $clone->post_name = substr($clone->post_name, 0, -9); | |
| 6810 | + } | |
| 6811 | + $url = get_permalink($clone); | |
| 6812 | + if (!$url) { | |
| 6813 | + continue; | |
| 6814 | + } | |
| 6815 | + | |
| 6816 | + // Local-table row count is exact in WordPress-DB mode; in Pinecone mode it | |
| 6817 | + // reads 0 but the delete below still routes to Pinecone and is idempotent. | |
| 6818 | + $local_rows = (int) $wpdb->get_var($wpdb->prepare( | |
| 6819 | + "SELECT COUNT(*) FROM {$table} WHERE source_url = %s", $url | |
| 6820 | + )); | |
| 6821 | + | |
| 6822 | + if ($dry_run) { | |
| 6823 | + if ($local_rows > 0) { | |
| 6824 | + WP_CLI::log(sprintf('Would remove %d row(s): %s (post %d, %s)', $local_rows, $url, $post_id, $post->post_status)); | |
| 6825 | + $pruned += $local_rows; | |
| 6826 | + } | |
| 6827 | + continue; | |
| 6828 | + } | |
| 6829 | + | |
| 6830 | + MxChat_Utils::delete_chunks_for_url($url, 'default'); | |
| 6831 | + if ($local_rows > 0) { | |
| 6832 | + WP_CLI::log(sprintf('Removed %d row(s): %s (post %d, %s)', $local_rows, $url, $post_id, $post->post_status)); | |
| 6833 | + $pruned += $local_rows; | |
| 6834 | + } | |
| 6835 | + } | |
| 6836 | + $more = $paged < $query->max_num_pages; | |
| 6837 | + $paged++; | |
| 6838 | + } while ($more); | |
| 6839 | + | |
| 6840 | + WP_CLI::success(sprintf( | |
| 6841 | + '%s %d local knowledge row(s) across %d non-published post(s) scanned.%s', | |
| 6842 | + $dry_run ? 'Would remove' : 'Removed', | |
| 6843 | + $pruned, | |
| 6844 | + $scanned, | |
| 6845 | + ' (Pinecone-mode deletions are not counted locally.)' | |
| 6846 | + )); | |
| 6847 | +} | |
| 6848 | + | |
| 6849 | +/** | |
| 6850 | + * WP-CLI: repair knowledge-base rows whose PDF text was imported in visual | |
| 6851 | + * (reversed) order before the RTL normalizer existed. 32bf9e fixed new | |
| 6852 | + * imports only; this fixes rows already in the table without the customer | |
| 6853 | + * having to re-source and re-upload the original PDFs (plan d1e6f7). | |
| 6854 | + * | |
| 6855 | + * Detection reuses MxChat_Utils::normalize_pdf_rtl() on the stored text: a | |
| 6856 | + * row is a candidate exactly when the normalizer would change it, so the | |
| 6857 | + * import-time heuristic and the repair heuristic can never disagree. | |
| 6858 | + * Repaired rows are RE-EMBEDDED — the stored vector was computed over | |
| 6859 | + * reversed text and is as broken as the text — so a wet run calls the | |
| 6860 | + * embedding provider once per repaired row on the site's API key. Runs | |
| 6861 | + * beyond 25 rows therefore require --yes. | |
| 6862 | + * | |
| 6863 | + * Scope notes: | |
| 6864 | + * - Scans the WordPress knowledge table. Pinecone-mode entries live in | |
| 6865 | + * Pinecone, not this table, and are not scanned; if a scanned row's bot | |
| 6866 | + * ALSO has Pinecone enabled (hybrid drift), the repaired entry is | |
| 6867 | + * re-submitted through the normal import path so the md5-keyed Pinecone | |
| 6868 | + * vector is replaced too. | |
| 6869 | + * - Knowledge rows do not carry a bot id; --bot only selects whose | |
| 6870 | + * embedding configuration (model + key) is used for re-embedding. | |
| 6871 | + * - The mxchat_pdf_rtl_normalize filter is honoured: a site that disabled | |
| 6872 | + * normalization gets detections of zero, not surprise rewrites. | |
| 6873 | + * - The metadata header the PDF importer stores before the text separator | |
| 6874 | + * is preserved byte-identical; only the text segment is repaired. | |
| 6875 | + * | |
| 6876 | + * ## OPTIONS | |
| 6877 | + * | |
| 6878 | + * [--dry-run] | |
| 6879 | + * : List the rows that would be repaired without changing anything. | |
| 6880 | + * | |
| 6881 | + * [--bot=<id>] | |
| 6882 | + * : Embedding configuration to use for re-embedding. Default: default. | |
| 6883 | + * | |
| 6884 | + * [--all-content] | |
| 6885 | + * : Scan every row containing right-to-left text, not just rows with PDF | |
| 6886 | + * provenance (a page anchor in the source URL, or pdf content type). | |
| 6887 | + * | |
| 6888 | + * [--yes] | |
| 6889 | + * : Proceed even when more than 25 rows need re-embedding (API cost gate). | |
| 6890 | + * | |
| 6891 | + * ## EXAMPLES | |
| 6892 | + * | |
| 6893 | + * wp mxchat rtl-repair --dry-run | |
| 6894 | + * wp mxchat rtl-repair | |
| 6895 | + * wp mxchat rtl-repair --all-content --yes | |
| 6896 | + */ | |
| 6897 | +public function cli_rtl_repair($args, $assoc_args) { | |
| 6898 | + global $wpdb; | |
| 6899 | + $dry_run = !empty($assoc_args['dry-run']); | |
| 6900 | + $all = !empty($assoc_args['all-content']); | |
| 6901 | + $yes = !empty($assoc_args['yes']); | |
| 6902 | + $bot_id = isset($assoc_args['bot']) ? sanitize_key($assoc_args['bot']) : 'default'; | |
| 6903 | + $table = $wpdb->prefix . 'mxchat_system_prompt_content'; | |
| 6904 | + | |
| 6905 | + // Detection pass — no API calls. Walk the table in id batches so a large | |
| 6906 | + // knowledge base never loads at once. | |
| 6907 | + $rtl_re = '/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u'; | |
| 6908 | + $candidates = array(); | |
| 6909 | + $scanned = 0; | |
| 6910 | + $last_id = 0; | |
| 6911 | + do { | |
| 6912 | + if ($all) { | |
| 6913 | + $rows = $wpdb->get_results($wpdb->prepare( | |
| 6914 | + "SELECT id, article_content, source_url, content_type FROM {$table} | |
| 6915 | + WHERE id > %d ORDER BY id ASC LIMIT 200", | |
| 6916 | + $last_id | |
| 6917 | + )); | |
| 6918 | + } else { | |
| 6919 | + $rows = $wpdb->get_results($wpdb->prepare( | |
| 6920 | + "SELECT id, article_content, source_url, content_type FROM {$table} | |
| 6921 | + WHERE id > %d AND (source_url LIKE %s OR content_type = 'pdf') | |
| 6922 | + ORDER BY id ASC LIMIT 200", | |
| 6923 | + $last_id, | |
| 6924 | + '%' . $wpdb->esc_like('#page=') . '%' | |
| 6925 | + )); | |
| 6926 | + } | |
| 6927 | + foreach ($rows as $row) { | |
| 6928 | + $last_id = (int) $row->id; | |
| 6929 | + $scanned++; | |
| 6930 | + $content = (string) $row->article_content; | |
| 6931 | + if (!preg_match($rtl_re, $content)) { | |
| 6932 | + continue; | |
| 6933 | + } | |
| 6934 | + list($header, $text) = $this->mxchat_rtl_repair_split($content); | |
| 6935 | + $normalized = MxChat_Utils::normalize_pdf_rtl($text, 'rtl-repair row ' . $row->id); | |
| 6936 | + if (is_string($normalized) && $normalized !== $text) { | |
| 6937 | + $candidates[] = array( | |
| 6938 | + 'id' => (int) $row->id, | |
| 6939 | + 'source_url' => (string) $row->source_url, | |
| 6940 | + 'content_type' => (string) $row->content_type, | |
| 6941 | + 'new_content' => $header . $normalized, | |
| 6942 | + ); | |
| 6943 | + } | |
| 6944 | + } | |
| 6945 | + } while (count($rows) === 200); | |
| 6946 | + | |
| 6947 | + WP_CLI::log(sprintf('Scanned %d row(s); %d stored in reversed (visual) order.', $scanned, count($candidates))); | |
| 6948 | + if (empty($candidates)) { | |
| 6949 | + WP_CLI::success('No reversed RTL rows found — nothing to repair.'); | |
| 6950 | + return; | |
| 6951 | + } | |
| 6952 | + | |
| 6953 | + foreach ($candidates as $c) { | |
| 6954 | + WP_CLI::log(sprintf('%s row %d %s', $dry_run ? 'Would repair' : 'Will repair', $c['id'], $c['source_url'])); | |
| 6955 | + } | |
| 6956 | + if ($dry_run) { | |
| 6957 | + WP_CLI::success(sprintf('Dry run: %d row(s) would be repaired and re-embedded. Run without --dry-run to apply.', count($candidates))); | |
| 6958 | + return; | |
| 6959 | + } | |
| 6960 | + | |
| 6961 | + // Cost gate: re-embedding spends the customer's API budget. | |
| 6962 | + WP_CLI::log(sprintf('Re-embedding will call the embedding provider once per row — %d call(s) on this site\'s API key.', count($candidates))); | |
| 6963 | + if (count($candidates) > 25 && !$yes) { | |
| 6964 | + WP_CLI::error(sprintf('%d rows need re-embedding (more than 25). Re-run with --yes to confirm the API cost. No rows were changed.', count($candidates))); | |
| 6965 | + } | |
| 6966 | + | |
| 6967 | + $bot_options = $this->get_bot_options($bot_id); | |
| 6968 | + $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options'); | |
| 6969 | + $preflight = MxChat_Utils::embedding_preflight($options); | |
| 6970 | + if (!$preflight['ok']) { | |
| 6971 | + WP_CLI::error('Embedding configuration problem: ' . $preflight['reason']); | |
| 6972 | + } | |
| 6973 | + $api_key = $preflight['api_key']; | |
| 6974 | + | |
| 6975 | + $pinecone_hybrid = $this->mxchat_rtl_repair_pinecone_enabled($bot_id); | |
| 6976 | + $repaired = 0; | |
| 6977 | + $failed = 0; | |
| 6978 | + foreach ($candidates as $c) { | |
| 6979 | + $vector = MxChat_Utils::regenerate_embedding($c['new_content'], $api_key, $bot_id); | |
| 6980 | + if (!is_array($vector)) { | |
| 6981 | + $failed++; | |
| 6982 | + $reason = is_wp_error($vector) ? $vector->get_error_message() : 'embedding request failed'; | |
| 6983 | + // Text and vector must stay consistent: never write repaired text | |
| 6984 | + // beside the stale reversed-text vector. | |
| 6985 | + WP_CLI::warning(sprintf('Row %d NOT repaired — %s. Row left unchanged.', $c['id'], $reason)); | |
| 6986 | + continue; | |
| 6987 | + } | |
| 6988 | + $wpdb->update( | |
| 6989 | + $table, | |
| 6990 | + array( | |
| 6991 | + 'article_content' => $c['new_content'], | |
| 6992 | + 'embedding_vector' => maybe_serialize($vector), | |
| 6993 | + ), | |
| 6994 | + array('id' => $c['id']), | |
| 6995 | + array('%s', '%s'), | |
| 6996 | + array('%d') | |
| 6997 | + ); | |
| 6998 | + $repaired++; | |
| 6999 | + if (class_exists('MxChat_Admin')) { | |
| 7000 | + MxChat_Admin::mxchat_log_debug('pdf_rtl_repaired', 'Stored KB row restored to logical order and re-embedded', array( | |
| 7001 | + 'row_id' => $c['id'], | |
| 7002 | + 'source_url' => $c['source_url'], | |
| 7003 | + 'bot' => $bot_id, | |
| 7004 | + )); | |
| 7005 | + } | |
| 7006 | + // Hybrid drift: the bot indexes into Pinecone but this row sat in the | |
| 7007 | + // WP table — push the repaired entry through the normal import path so | |
| 7008 | + // the md5(source_url)-keyed Pinecone vector is replaced as well. | |
| 7009 | + if ($pinecone_hybrid) { | |
| 7010 | + MxChat_Utils::submit_content_to_db( | |
| 7011 | + $c['new_content'], | |
| 7012 | + $c['source_url'], | |
| 7013 | + $api_key, | |
| 7014 | + null, | |
| 7015 | + $bot_id, | |
| 7016 | + $c['content_type'] !== '' ? $c['content_type'] : 'pdf' | |
| 7017 | + ); | |
| 7018 | + } | |
| 7019 | + } | |
| 7020 | + | |
| 7021 | + WP_CLI::success(sprintf('Repaired + re-embedded %d row(s); %d failed; %d scanned.', $repaired, $failed, $scanned)); | |
| 7022 | +} | |
| 7023 | + | |
| 7024 | +/** | |
| 7025 | + * Split a stored KB row into (metadata header incl. separator, text segment). | |
| 7026 | + * The PDF importer stores wp_json_encode($metadata) . "\n---\n" . $text — | |
| 7027 | + * repair must touch only the text and keep the header byte-identical. | |
| 7028 | + */ | |
| 7029 | +private function mxchat_rtl_repair_split($content) { | |
| 7030 | + $sep = "\n---\n"; | |
| 7031 | + $pos = strpos($content, $sep); | |
| 7032 | + if ($pos !== false && $pos > 0 && $content[0] === '{') { | |
| 7033 | + $maybe_json = substr($content, 0, $pos); | |
| 7034 | + if (json_decode($maybe_json) !== null) { | |
| 7035 | + return array(substr($content, 0, $pos + strlen($sep)), substr($content, $pos + strlen($sep))); | |
| 7036 | + } | |
| 7037 | + } | |
| 7038 | + return array('', $content); | |
| 7039 | +} | |
| 7040 | + | |
| 7041 | +/** | |
| 7042 | + * Mirror of MxChat_Utils::is_pinecone_enabled_for_bot() (private there) for | |
| 7043 | + * the repair CLI's hybrid-drift check. | |
| 7044 | + */ | |
| 7045 | +private function mxchat_rtl_repair_pinecone_enabled($bot_id) { | |
| 7046 | + if ($bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager')) { | |
| 7047 | + $cfg = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id); | |
| 7048 | + if (!empty($cfg)) { | |
| 7049 | + return !empty($cfg['use_pinecone']) && !empty($cfg['api_key']) && !empty($cfg['host']); | |
| 7050 | + } | |
| 7051 | + } | |
| 7052 | + $po = get_option('mxchat_pinecone_addon_options'); | |
| 7053 | + return !empty($po['mxchat_use_pinecone']) && $po['mxchat_use_pinecone'] !== '0' | |
| 7054 | + && !empty($po['mxchat_pinecone_api_key']) && !empty($po['mxchat_pinecone_host']); | |
| 7055 | +} | |
| 7056 | + | |
| 5913 | 7057 | public function mxchat_handle_post_delete($post_id) { |
| 5914 | 7058 | // Get post data before it's deleted |
| 5915 | 7059 | $post = get_post($post_id); |
| 5916 | 7060 | |
| @@ -6010,110 +7154,20 @@ | ||
| 6010 | 7154 | |
| 6011 | 7155 | $source_url = get_permalink($product->get_id()); |
| 6012 | 7156 | $product_id = $product->get_id(); |
| 6013 | 7157 | |
| 6014 | - // Build product content | |
| 6015 | - $title = $product->get_name(); | |
| 6016 | - $description = $product->get_description(); | |
| 6017 | - $short_description = $product->get_short_description(); | |
| 6018 | - $regular_price = $product->get_regular_price(); | |
| 6019 | - $sale_price = $product->get_sale_price(); | |
| 6020 | - $price = $product->get_price(); | |
| 6021 | - $sku = $product->get_sku(); | |
| 7158 | + // Build product content via the shared WC-object assembler (a3d60c) — this | |
| 7159 | + // writer owns product rows whenever the integration is on. | |
| 7160 | + $content = $this->mxchat_prepare_product_content_for_indexing($product); | |
| 6022 | 7161 | |
| 6023 | - // Get currency symbol | |
| 6024 | - $currency_symbol = get_woocommerce_currency_symbol(); | |
| 6025 | - | |
| 6026 | - // Format content consistently | |
| 6027 | - $content = $title . "\n\n"; | |
| 6028 | - | |
| 6029 | - if (!empty($short_description)) { | |
| 6030 | - $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n"; | |
| 6031 | - } | |
| 6032 | - | |
| 6033 | - if (!empty($description)) { | |
| 6034 | - $content .= wp_strip_all_tags($description) . "\n\n"; | |
| 6035 | - } | |
| 6036 | - | |
| 6037 | - // Add pricing information | |
| 6038 | - if (!empty($regular_price)) { | |
| 6039 | - $content .= "Price: " . $currency_symbol . $regular_price . "\n"; | |
| 6040 | - } elseif (!empty($price)) { | |
| 6041 | - $content .= "Price: " . $currency_symbol . $price . "\n"; | |
| 6042 | - } | |
| 6043 | - | |
| 6044 | - if (!empty($sale_price) && $sale_price !== $regular_price) { | |
| 6045 | - $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n"; | |
| 6046 | - } | |
| 6047 | - | |
| 6048 | - // Handle variable products - show price range | |
| 6049 | - if ($product->is_type('variable')) { | |
| 6050 | - $min_price = $product->get_variation_price('min'); | |
| 6051 | - $max_price = $product->get_variation_price('max'); | |
| 6052 | - if ($min_price !== $max_price) { | |
| 6053 | - $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n"; | |
| 6054 | - } | |
| 6055 | - } | |
| 6056 | - | |
| 6057 | - if (!empty($sku)) { | |
| 6058 | - $content .= "SKU: " . $sku . "\n"; | |
| 6059 | - } | |
| 6060 | - | |
| 6061 | - // Get product categories | |
| 6062 | - $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names')); | |
| 6063 | - if (!empty($categories) && !is_wp_error($categories)) { | |
| 6064 | - $content .= "Categories: " . implode(', ', $categories) . "\n"; | |
| 6065 | - } | |
| 6066 | - | |
| 6067 | - // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots) | |
| 6068 | - $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true); | |
| 6069 | - if (!empty($custom_tabs) && is_array($custom_tabs)) { | |
| 6070 | - foreach ($custom_tabs as $tab) { | |
| 6071 | - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : ''); | |
| 6072 | - $tab_content = isset($tab['content']) ? $tab['content'] : ''; | |
| 6073 | - | |
| 6074 | - if (!empty($tab_title) && !empty($tab_content)) { | |
| 6075 | - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n"; | |
| 6076 | - } | |
| 6077 | - } | |
| 6078 | - } | |
| 6079 | - | |
| 6080 | - // Also check for reusable/saved tabs applied to this product | |
| 6081 | - $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true); | |
| 6082 | - if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) { | |
| 6083 | - // Get the saved tabs option | |
| 6084 | - $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array()); | |
| 6085 | - if (!empty($saved_tabs) && is_array($saved_tabs)) { | |
| 6086 | - foreach ($applied_saved_tabs as $saved_tab_id) { | |
| 6087 | - if (isset($saved_tabs[$saved_tab_id])) { | |
| 6088 | - $tab = $saved_tabs[$saved_tab_id]; | |
| 6089 | - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : ''); | |
| 6090 | - $tab_content = isset($tab['content']) ? $tab['content'] : ''; | |
| 6091 | - | |
| 6092 | - if (!empty($tab_title) && !empty($tab_content)) { | |
| 6093 | - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n"; | |
| 6094 | - } | |
| 6095 | - } | |
| 6096 | - } | |
| 6097 | - } | |
| 6098 | - } | |
| 6099 | - | |
| 6100 | - // Get API key with proper model detection | |
| 6101 | - $options = get_option('mxchat_options'); | |
| 6102 | - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002'; | |
| 6103 | - | |
| 6104 | - if (strpos($selected_model, 'voyage') === 0) { | |
| 6105 | - $api_key = $options['voyage_api_key'] ?? ''; | |
| 6106 | - } elseif (strpos($selected_model, 'gemini-embedding') === 0) { | |
| 6107 | - $api_key = $options['gemini_api_key'] ?? ''; | |
| 6108 | - } else { | |
| 6109 | - $api_key = $options['api_key'] ?? ''; | |
| 6110 | - } | |
| 6111 | - | |
| 6112 | - if (empty($api_key)) { | |
| 6113 | - //error_log('MxChat Auto-sync: No API key configured for embedding model'); | |
| 7162 | + // Embedding decision — custom-provider-aware (plan cbd5fd); silent-return | |
| 7163 | + // shape preserved. | |
| 7164 | + $preflight = MxChat_Utils::embedding_preflight(get_option('mxchat_options')); | |
| 7165 | + if (!$preflight['ok']) { | |
| 7166 | + //error_log('MxChat Auto-sync: embedding pre-flight failed: ' . $preflight['reason']); | |
| 6114 | 7167 | return; |
| 6115 | 7168 | } |
| 7169 | + $api_key = $preflight['api_key']; | |
| 6116 | 7170 | |
| 6117 | 7171 | // Use the centralized utility function for storage |
| 6118 | 7172 | $result = MxChat_Utils::submit_content_to_db( |
| 6119 | 7173 | $content, |
| @@ -6189,17 +7243,23 @@ | ||
| 6189 | 7243 | |
| 6190 | 7244 | // Delete from Pinecone |
| 6191 | 7245 | $pinecone_manager = MxChat_Pinecone_Manager::get_instance(); |
| 6192 | 7246 | $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id( |
| 6193 | - $vector_id, | |
| 6194 | - $pinecone_options['mxchat_pinecone_api_key'], | |
| 6195 | - $pinecone_options['mxchat_pinecone_host'] | |
| 7247 | + $vector_id, | |
| 7248 | + $pinecone_options['mxchat_pinecone_api_key'], | |
| 7249 | + $pinecone_options['mxchat_pinecone_host'], | |
| 7250 | + $pinecone_options['mxchat_pinecone_namespace'] ?? '' | |
| 6196 | 7251 | ); |
| 6197 | 7252 | |
| 6198 | 7253 | if ($result['success']) { |
| 7254 | + // Mirror the removal to the OpenAI Vector Store mapping (plan 15b5c6); | |
| 7255 | + // a chunk vector id reduces to its base entry there. | |
| 7256 | + if (class_exists('MxChat_Vectorstore_Manager')) { | |
| 7257 | + MxChat_Vectorstore_Manager::sync_delete_by_key($vector_id, 'default'); | |
| 7258 | + } | |
| 6199 | 7259 | // No cache clearing needed since we removed caching |
| 6200 | - set_transient('mxchat_admin_notice_success', | |
| 6201 | - esc_html__('Entry deleted successfully from Pinecone.', 'mxchat'), | |
| 7260 | + set_transient('mxchat_admin_notice_success', | |
| 7261 | + esc_html__('Entry deleted successfully from Pinecone.', 'mxchat'), | |
| 6202 | 7262 | 30 |
| 6203 | 7263 | ); |
| 6204 | 7264 | } else { |
| 6205 | 7265 | set_transient('mxchat_admin_notice_error', |
| @@ -6244,16 +7304,21 @@ | ||
| 6244 | 7304 | wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id); |
| 6245 | 7305 | exit; |
| 6246 | 7306 | } |
| 6247 | 7307 | |
| 6248 | - // Delete from the correct Pinecone index | |
| 7308 | + // Delete from the correct Pinecone index and namespace | |
| 6249 | 7309 | $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id( |
| 6250 | - $vector_id, | |
| 6251 | - $pinecone_options['mxchat_pinecone_api_key'], | |
| 6252 | - $pinecone_options['mxchat_pinecone_host'] | |
| 7310 | + $vector_id, | |
| 7311 | + $pinecone_options['mxchat_pinecone_api_key'], | |
| 7312 | + $pinecone_options['mxchat_pinecone_host'], | |
| 7313 | + $pinecone_options['mxchat_pinecone_namespace'] ?? '' | |
| 6253 | 7314 | ); |
| 6254 | 7315 | |
| 6255 | 7316 | if ($result['success']) { |
| 7317 | + // Mirror the removal to the OpenAI Vector Store mapping (plan 15b5c6) | |
| 7318 | + if (class_exists('MxChat_Vectorstore_Manager')) { | |
| 7319 | + MxChat_Vectorstore_Manager::sync_delete_by_key($vector_id, $bot_id); | |
| 7320 | + } | |
| 6256 | 7321 | // No cache clearing needed since we removed caching |
| 6257 | 7322 | wp_send_json_success(array( |
| 6258 | 7323 | 'message' => 'Entry deleted successfully from Pinecone', |
| 6259 | 7324 | 'vector_id' => $vector_id, |
| @@ -6352,8 +7417,13 @@ | ||
| 6352 | 7417 | } |
| 6353 | 7418 | } |
| 6354 | 7419 | |
| 6355 | 7420 | if (empty($vectors_to_delete)) { |
| 7421 | + // Entry already gone from Pinecone — still clear any mirrored | |
| 7422 | + // Vector Store file so it can't outlive the entry (plan 15b5c6). | |
| 7423 | + if (class_exists('MxChat_Vectorstore_Manager')) { | |
| 7424 | + MxChat_Vectorstore_Manager::sync_delete_entry($source_url, $bot_id); | |
| 7425 | + } | |
| 6356 | 7426 | wp_send_json_success(array( |
| 6357 | 7427 | 'message' => 'No vectors found to delete', |
| 6358 | 7428 | 'source_url' => $source_url |
| 6359 | 7429 | )); |
| @@ -6394,8 +7464,13 @@ | ||
| 6394 | 7464 | wp_send_json_error('Pinecone API error (HTTP ' . $response_code . ')'); |
| 6395 | 7465 | exit; |
| 6396 | 7466 | } |
| 6397 | 7467 | |
| 7468 | + // Mirror the removal to the OpenAI Vector Store (plan 15b5c6) | |
| 7469 | + if (class_exists('MxChat_Vectorstore_Manager')) { | |
| 7470 | + MxChat_Vectorstore_Manager::sync_delete_entry($source_url, $bot_id); | |
| 7471 | + } | |
| 7472 | + | |
| 6398 | 7473 | wp_send_json_success(array( |
| 6399 | 7474 | 'message' => 'All chunks deleted successfully from Pinecone', |
| 6400 | 7475 | 'source_url' => $source_url, |
| 6401 | 7476 | 'deleted_count' => count($vectors_to_delete) |
| @@ -6417,8 +7492,13 @@ | ||
| 6417 | 7492 | wp_send_json_error('Failed to delete from database: ' . $wpdb->last_error); |
| 6418 | 7493 | exit; |
| 6419 | 7494 | } |
| 6420 | 7495 | |
| 7496 | + // Mirror the removal to the OpenAI Vector Store (plan 15b5c6) | |
| 7497 | + if (class_exists('MxChat_Vectorstore_Manager')) { | |
| 7498 | + MxChat_Vectorstore_Manager::sync_delete_entry($source_url, $bot_id); | |
| 7499 | + } | |
| 7500 | + | |
| 6421 | 7501 | wp_send_json_success(array( |
| 6422 | 7502 | 'message' => 'All chunks deleted successfully from database', |
| 6423 | 7503 | 'source_url' => $source_url, |
| 6424 | 7504 | 'deleted_count' => $result |
| @@ -6453,8 +7533,15 @@ | ||
| 6453 | 7533 | |
| 6454 | 7534 | global $wpdb; |
| 6455 | 7535 | $table_name = $wpdb->prefix . 'mxchat_system_prompt_content'; |
| 6456 | 7536 | |
| 7537 | + // Capture the identity BEFORE the row disappears — needed to mirror the | |
| 7538 | + // change into the Vector Store (plan 15b5c6). | |
| 7539 | + $source_url = $wpdb->get_var($wpdb->prepare( | |
| 7540 | + "SELECT source_url FROM {$table_name} WHERE id = %d", | |
| 7541 | + $entry_id | |
| 7542 | + )); | |
| 7543 | + | |
| 6457 | 7544 | // Clear cache for this entry |
| 6458 | 7545 | wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts'); |
| 6459 | 7546 | |
| 6460 | 7547 | // Delete from database |
| @@ -6464,8 +7551,23 @@ | ||
| 6464 | 7551 | array('%d') |
| 6465 | 7552 | ); |
| 6466 | 7553 | |
| 6467 | 7554 | if ($result !== false) { |
| 7555 | + // Mirror to the Vector Store: if sibling rows remain (this was one | |
| 7556 | + // chunk of a larger entry) the entry's file is REFRESHED from what's | |
| 7557 | + // left; if none remain, the file is removed. | |
| 7558 | + if (!empty($source_url) && class_exists('MxChat_Vectorstore_Manager')) { | |
| 7559 | + $remaining = (int) $wpdb->get_var($wpdb->prepare( | |
| 7560 | + "SELECT COUNT(*) FROM {$table_name} WHERE source_url = %s", | |
| 7561 | + $source_url | |
| 7562 | + )); | |
| 7563 | + if ($remaining > 0) { | |
| 7564 | + MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, '', 'default'); | |
| 7565 | + } else { | |
| 7566 | + MxChat_Vectorstore_Manager::sync_delete_entry($source_url, 'default'); | |
| 7567 | + } | |
| 7568 | + } | |
| 7569 | + | |
| 6468 | 7570 | wp_send_json_success(array( |
| 6469 | 7571 | 'message' => 'Entry deleted successfully', |
| 6470 | 7572 | 'entry_id' => $entry_id |
| 6471 | 7573 | )); |
| @@ -6518,8 +7620,9 @@ | ||
| 6518 | 7620 | $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1'; |
| 6519 | 7621 | |
| 6520 | 7622 | $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? ''; |
| 6521 | 7623 | $host = $pinecone_options['mxchat_pinecone_host'] ?? ''; |
| 7624 | + $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? ''; | |
| 6522 | 7625 | |
| 6523 | 7626 | // ============================================= |
| 6524 | 7627 | // PHASE 1: Collect all Pinecone vector IDs |
| 6525 | 7628 | // and separate WordPress entries |
| @@ -6526,8 +7629,10 @@ | ||
| 6526 | 7629 | // ============================================= |
| 6527 | 7630 | $pinecone_entry_ids = array(); // entry IDs that are pinecone-sourced |
| 6528 | 7631 | $wordpress_entries = array(); // entries for WordPress DB deletion |
| 6529 | 7632 | $all_vector_ids = array(); // all pinecone vector IDs to delete in one batch |
| 7633 | + $vs_mirror_urls = array(); // Vector Store mirror: URLs to delete (plan 15b5c6) | |
| 7634 | + $vs_mirror_keys = array(); // Vector Store mirror: bare vector ids to delete | |
| 6530 | 7635 | |
| 6531 | 7636 | foreach ($entries as $entry) { |
| 6532 | 7637 | $entry_id = sanitize_text_field($entry['id'] ?? ''); |
| 6533 | 7638 | $source = sanitize_text_field($entry['source'] ?? 'wordpress'); |
| @@ -6552,8 +7657,11 @@ | ||
| 6552 | 7657 | $base_vector_id = md5($source_url); |
| 6553 | 7658 | $all_vector_ids[] = $base_vector_id; |
| 6554 | 7659 | |
| 6555 | 7660 | $list_url = 'https://' . $host . '/vectors/list?prefix=' . urlencode($base_vector_id . '_chunk_') . '&limit=100'; |
| 7661 | + if (!empty($namespace)) { | |
| 7662 | + $list_url .= '&namespace=' . rawurlencode($namespace); | |
| 7663 | + } | |
| 6556 | 7664 | $list_response = wp_remote_get($list_url, array( |
| 6557 | 7665 | 'headers' => array( |
| 6558 | 7666 | 'Api-Key' => $api_key, |
| 6559 | 7667 | 'accept' => 'application/json' |
| @@ -6574,8 +7682,14 @@ | ||
| 6574 | 7682 | } else { |
| 6575 | 7683 | // Single entry: the entry_id IS the vector ID |
| 6576 | 7684 | $all_vector_ids[] = $entry_id; |
| 6577 | 7685 | } |
| 7686 | + | |
| 7687 | + if (!empty($source_url)) { | |
| 7688 | + $vs_mirror_urls[] = $source_url; | |
| 7689 | + } else { | |
| 7690 | + $vs_mirror_keys[] = $entry_id; | |
| 7691 | + } | |
| 6578 | 7692 | } else { |
| 6579 | 7693 | $wordpress_entries[] = $entry; |
| 6580 | 7694 | } |
| 6581 | 7695 | } |
| @@ -6588,8 +7702,12 @@ | ||
| 6588 | 7702 | $pinecone_success = true; |
| 6589 | 7703 | $batches = array_chunk($all_vector_ids, 100); |
| 6590 | 7704 | |
| 6591 | 7705 | foreach ($batches as $batch) { |
| 7706 | + $delete_body = array('ids' => $batch); | |
| 7707 | + if (!empty($namespace)) { | |
| 7708 | + $delete_body['namespace'] = $namespace; | |
| 7709 | + } | |
| 6592 | 7710 | $delete_response = wp_remote_post("https://{$host}/vectors/delete", array( |
| 6593 | 7711 | 'headers' => array( |
| 6594 | 7712 | 'Api-Key' => $api_key, |
| 6595 | 7713 | 'accept' => 'application/json', |
| @@ -6594,9 +7712,9 @@ | ||
| 6594 | 7712 | 'Api-Key' => $api_key, |
| 6595 | 7713 | 'accept' => 'application/json', |
| 6596 | 7714 | 'content-type' => 'application/json' |
| 6597 | 7715 | ), |
| 6598 | - 'body' => wp_json_encode(array('ids' => $batch)), | |
| 7716 | + 'body' => wp_json_encode($delete_body), | |
| 6599 | 7717 | 'timeout' => 60 |
| 6600 | 7718 | )); |
| 6601 | 7719 | |
| 6602 | 7720 | if (is_wp_error($delete_response)) { |
| @@ -6621,8 +7739,18 @@ | ||
| 6621 | 7739 | } else { |
| 6622 | 7740 | $failed_ids[] = $eid; |
| 6623 | 7741 | } |
| 6624 | 7742 | } |
| 7743 | + | |
| 7744 | + // Mirror the removals to the OpenAI Vector Store (plan 15b5c6) | |
| 7745 | + if ($pinecone_success && class_exists('MxChat_Vectorstore_Manager')) { | |
| 7746 | + foreach (array_unique($vs_mirror_urls) as $vs_url) { | |
| 7747 | + MxChat_Vectorstore_Manager::sync_delete_entry($vs_url, $bot_id); | |
| 7748 | + } | |
| 7749 | + foreach (array_unique($vs_mirror_keys) as $vs_key) { | |
| 7750 | + MxChat_Vectorstore_Manager::sync_delete_by_key($vs_key, $bot_id); | |
| 7751 | + } | |
| 7752 | + } | |
| 6625 | 7753 | } |
| 6626 | 7754 | |
| 6627 | 7755 | // ============================================= |
| 6628 | 7756 | // PHASE 3: WordPress database deletions |
| @@ -6642,9 +7770,15 @@ | ||
| 6642 | 7770 | $table_name, |
| 6643 | 7771 | array('source_url' => $source_url), |
| 6644 | 7772 | array('%s') |
| 6645 | 7773 | ); |
| 7774 | + $row_url = $source_url; | |
| 6646 | 7775 | } else { |
| 7776 | + // Identity captured pre-delete for the Vector Store mirror | |
| 7777 | + $row_url = $wpdb->get_var($wpdb->prepare( | |
| 7778 | + "SELECT source_url FROM {$table_name} WHERE id = %d", | |
| 7779 | + intval($entry_id) | |
| 7780 | + )); | |
| 6647 | 7781 | wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts'); |
| 6648 | 7782 | $result = $wpdb->delete( |
| 6649 | 7783 | $table_name, |
| 6650 | 7784 | array('id' => intval($entry_id)), |
| @@ -6653,8 +7787,21 @@ | ||
| 6653 | 7787 | } |
| 6654 | 7788 | |
| 6655 | 7789 | if ($result !== false) { |
| 6656 | 7790 | $success_ids[] = $entry_id; |
| 7791 | + // Mirror to the Vector Store: refresh the entry's file when | |
| 7792 | + // sibling chunk rows survive, remove it when none do. | |
| 7793 | + if (!empty($row_url) && class_exists('MxChat_Vectorstore_Manager')) { | |
| 7794 | + $remaining = (int) $wpdb->get_var($wpdb->prepare( | |
| 7795 | + "SELECT COUNT(*) FROM {$table_name} WHERE source_url = %s", | |
| 7796 | + $row_url | |
| 7797 | + )); | |
| 7798 | + if ($remaining > 0) { | |
| 7799 | + MxChat_Vectorstore_Manager::sync_upsert_entry($row_url, '', $bot_id); | |
| 7800 | + } else { | |
| 7801 | + MxChat_Vectorstore_Manager::sync_delete_entry($row_url, $bot_id); | |
| 7802 | + } | |
| 7803 | + } | |
| 6657 | 7804 | } else { |
| 6658 | 7805 | $failed_ids[] = $entry_id; |
| 6659 | 7806 | $errors[] = "Database error for entry: $entry_id"; |
| 6660 | 7807 | } |
| @@ -6806,9 +7953,29 @@ | ||
| 6806 | 7953 | if ($result === false) { |
| 6807 | 7954 | wp_send_json_error('Database update failed: ' . $wpdb->last_error); |
| 6808 | 7955 | exit; |
| 6809 | 7956 | } |
| 6810 | - | |
| 7957 | + | |
| 7958 | + // Keep the OpenAI Vector Store mirror consistent with the new restriction | |
| 7959 | + // (plan 15b5c6): non-public pulls the mirrored file, public re-mirrors. | |
| 7960 | + if (class_exists('MxChat_Vectorstore_Manager')) { | |
| 7961 | + if ($data_source === 'pinecone') { | |
| 7962 | + if ($role_restriction !== 'public') { | |
| 7963 | + MxChat_Vectorstore_Manager::sync_delete_by_key($entry_id, 'default'); | |
| 7964 | + } | |
| 7965 | + // Public again: Pinecone-mode content isn't held locally, so the | |
| 7966 | + // entry re-mirrors on its next save/import rather than here. | |
| 7967 | + } else { | |
| 7968 | + $row_url = $wpdb->get_var($wpdb->prepare( | |
| 7969 | + "SELECT source_url FROM {$wpdb->prefix}mxchat_system_prompt_content WHERE id = %d", | |
| 7970 | + absint($entry_id) | |
| 7971 | + )); | |
| 7972 | + if (!empty($row_url)) { | |
| 7973 | + MxChat_Vectorstore_Manager::handle_role_change($row_url, 'default', $role_restriction); | |
| 7974 | + } | |
| 7975 | + } | |
| 7976 | + } | |
| 7977 | + | |
| 6811 | 7978 | wp_send_json_success(array( |
| 6812 | 7979 | 'message' => 'Role restriction updated successfully', |
| 6813 | 7980 | 'role_restriction' => $role_restriction, |
| 6814 | 7981 | 'data_source' => $data_source, |
| @@ -7160,9 +8327,9 @@ | ||
| 7160 | 8327 | ); |
| 7161 | 8328 | } else { |
| 7162 | 8329 | // Update WordPress DB |
| 7163 | 8330 | $table_name = $wpdb->prefix . 'mxchat_system_prompt_content'; |
| 7164 | - | |
| 8331 | + | |
| 7165 | 8332 | $wpdb->update( |
| 7166 | 8333 | $table_name, |
| 7167 | 8334 | array('role_restriction' => $highest_role), |
| 7168 | 8335 | array('source_url' => $source_url), |
| @@ -7169,8 +8336,15 @@ | ||
| 7169 | 8336 | array('%s'), |
| 7170 | 8337 | array('%s') |
| 7171 | 8338 | ); |
| 7172 | 8339 | } |
| 8340 | + | |
| 8341 | + // The entry's restriction just changed — keep the OpenAI Vector Store | |
| 8342 | + // mirror consistent: non-public pulls the file (file_search has no | |
| 8343 | + // per-role filtering), public re-mirrors it (plan 15b5c6). | |
| 8344 | + if (class_exists('MxChat_Vectorstore_Manager')) { | |
| 8345 | + MxChat_Vectorstore_Manager::handle_role_change($source_url, 'default', $highest_role); | |
| 8346 | + } | |
| 7173 | 8347 | } |
| 7174 | 8348 | |
| 7175 | 8349 | /** |
| 7176 | 8350 | * Apply role restriction after content is stored (for auto-sync) |
| @@ -7239,9 +8413,9 @@ | ||
| 7239 | 8413 | ); |
| 7240 | 8414 | } else { |
| 7241 | 8415 | // Update WordPress DB |
| 7242 | 8416 | $table_name = $wpdb->prefix . 'mxchat_system_prompt_content'; |
| 7243 | - | |
| 8417 | + | |
| 7244 | 8418 | $wpdb->update( |
| 7245 | 8419 | $table_name, |
| 7246 | 8420 | array('role_restriction' => $highest_role), |
| 7247 | 8421 | array('source_url' => $source_url), |
| @@ -7248,8 +8422,15 @@ | ||
| 7248 | 8422 | array('%s'), |
| 7249 | 8423 | array('%s') |
| 7250 | 8424 | ); |
| 7251 | 8425 | } |
| 8426 | + | |
| 8427 | + // The entry's restriction just changed — keep the OpenAI Vector Store | |
| 8428 | + // mirror consistent: non-public pulls the file (file_search has no | |
| 8429 | + // per-role filtering), public re-mirrors it (plan 15b5c6). | |
| 8430 | + if (class_exists('MxChat_Vectorstore_Manager')) { | |
| 8431 | + MxChat_Vectorstore_Manager::handle_role_change($source_url, 'default', $highest_role); | |
| 8432 | + } | |
| 7252 | 8433 | } |
| 7253 | 8434 | |
| 7254 | 8435 | |
| 7255 | 8436 | // ======================================== |
| @@ -7710,25 +8891,19 @@ | ||
| 7710 | 8891 | if (empty($url)) { |
| 7711 | 8892 | return new WP_Error('invalid_url', 'URL is empty'); |
| 7712 | 8893 | } |
| 7713 | 8894 | |
| 7714 | - // Get bot-specific API key early (needed for both paths) | |
| 8895 | + // Get bot-specific embedding decision early (needed for both paths) — | |
| 8896 | + // custom-provider-aware (plan cbd5fd). Error code preserved. | |
| 7715 | 8897 | $bot_options = $this->get_bot_options($bot_id); |
| 7716 | 8898 | $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options'); |
| 7717 | - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002'; | |
| 7718 | 8899 | |
| 7719 | - if (strpos($selected_model, 'voyage') === 0) { | |
| 7720 | - $api_key = $options['voyage_api_key'] ?? ''; | |
| 7721 | - } elseif (strpos($selected_model, 'gemini-embedding') === 0) { | |
| 7722 | - $api_key = $options['gemini_api_key'] ?? ''; | |
| 7723 | - } else { | |
| 7724 | - $api_key = $options['api_key'] ?? ''; | |
| 8900 | + $preflight = MxChat_Utils::embedding_preflight($options); | |
| 8901 | + if (!$preflight['ok']) { | |
| 8902 | + return new WP_Error('no_api_key', $preflight['reason'] . ' (bot: ' . $bot_id . ')'); | |
| 7725 | 8903 | } |
| 8904 | + $api_key = $preflight['api_key']; | |
| 7726 | 8905 | |
| 7727 | - if (empty($api_key)) { | |
| 7728 | - return new WP_Error('no_api_key', 'API key not configured for bot: ' . $bot_id); | |
| 7729 | - } | |
| 7730 | - | |
| 7731 | 8906 | // Check if this is a WooCommerce product URL and WooCommerce is active |
| 7732 | 8907 | $is_product_url = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false); |
| 7733 | 8908 | $content_type = $is_product_url ? 'product' : 'url'; |
| 7734 | 8909 | |
| @@ -7755,9 +8930,9 @@ | ||
| 7755 | 8930 | $is_likely_pdf = (strtolower(pathinfo(parse_url($url, PHP_URL_PATH) ?: '', PATHINFO_EXTENSION)) === 'pdf'); |
| 7756 | 8931 | $response = wp_remote_get($url, array( |
| 7757 | 8932 | 'timeout' => $is_likely_pdf ? 120 : 30, |
| 7758 | 8933 | 'redirection' => 5, |
| 7759 | - 'user-agent' => 'MxChat/1.0' | |
| 8934 | + 'user-agent' => mxchat_ingest_user_agent(), | |
| 7760 | 8935 | )); |
| 7761 | 8936 | |
| 7762 | 8937 | if (is_wp_error($response)) { |
| 7763 | 8938 | return $response; |
| @@ -7922,8 +9097,9 @@ | ||
| 7922 | 9097 | |
| 7923 | 9098 | for ($i = 0; $i < $total_pages; $i++) { |
| 7924 | 9099 | $page_num = $i + 1; |
| 7925 | 9100 | $text = $pages[$i]->getText(); |
| 9101 | + $text = MxChat_Utils::normalize_pdf_rtl($text, 'kb_pdf_import page ' . $page_num); | |
| 7926 | 9102 | if (empty($text)) { |
| 7927 | 9103 | $skipped_pages[] = 'Page ' . $page_num . ': No text could be extracted — page may contain only images, links, or non-standard encoding'; |
| 7928 | 9104 | continue; |
| 7929 | 9105 | } |
| @@ -8010,95 +9186,12 @@ | ||
| 8010 | 9186 | if (!$product) { |
| 8011 | 9187 | return false; |
| 8012 | 9188 | } |
| 8013 | 9189 | |
| 8014 | - // Build product content with pricing (similar to mxchat_store_product_embedding) | |
| 8015 | - $title = $product->get_name(); | |
| 8016 | - $description = $product->get_description(); | |
| 8017 | - $short_description = $product->get_short_description(); | |
| 8018 | - $sku = $product->get_sku(); | |
| 9190 | + // Build product content via the shared WC-object assembler (a3d60c) — same | |
| 9191 | + // body as the auto-sync product writer, so the two paths can never drift. | |
| 9192 | + $content = $this->mxchat_prepare_product_content_for_indexing($product); | |
| 8019 | 9193 | |
| 8020 | - // Get pricing information | |
| 8021 | - $regular_price = $product->get_regular_price(); | |
| 8022 | - $sale_price = $product->get_sale_price(); | |
| 8023 | - $price = $product->get_price(); // Current active price | |
| 8024 | - | |
| 8025 | - // Get currency symbol | |
| 8026 | - $currency_symbol = get_woocommerce_currency_symbol(); | |
| 8027 | - | |
| 8028 | - // Format content | |
| 8029 | - $content = $title . "\n\n"; | |
| 8030 | - | |
| 8031 | - if (!empty($short_description)) { | |
| 8032 | - $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n"; | |
| 8033 | - } | |
| 8034 | - | |
| 8035 | - if (!empty($description)) { | |
| 8036 | - $content .= wp_strip_all_tags($description) . "\n\n"; | |
| 8037 | - } | |
| 8038 | - | |
| 8039 | - // Add pricing information | |
| 8040 | - if (!empty($regular_price)) { | |
| 8041 | - $content .= "Price: " . $currency_symbol . $regular_price . "\n"; | |
| 8042 | - } elseif (!empty($price)) { | |
| 8043 | - $content .= "Price: " . $currency_symbol . $price . "\n"; | |
| 8044 | - } | |
| 8045 | - | |
| 8046 | - if (!empty($sale_price) && $sale_price !== $regular_price) { | |
| 8047 | - $content .= "Sale Price: " . $currency_symbol . $sale_price . "\n"; | |
| 8048 | - } | |
| 8049 | - | |
| 8050 | - // Handle variable products - show price range | |
| 8051 | - if ($product->is_type('variable')) { | |
| 8052 | - $min_price = $product->get_variation_price('min'); | |
| 8053 | - $max_price = $product->get_variation_price('max'); | |
| 8054 | - if ($min_price !== $max_price) { | |
| 8055 | - $content .= "Price Range: " . $currency_symbol . $min_price . " - " . $currency_symbol . $max_price . "\n"; | |
| 8056 | - } | |
| 8057 | - } | |
| 8058 | - | |
| 8059 | - if (!empty($sku)) { | |
| 8060 | - $content .= "SKU: " . $sku . "\n"; | |
| 8061 | - } | |
| 8062 | - | |
| 8063 | - // Get product categories | |
| 8064 | - $categories = wp_get_post_terms($product_id, 'product_cat', array('fields' => 'names')); | |
| 8065 | - if (!empty($categories) && !is_wp_error($categories)) { | |
| 8066 | - $content .= "Categories: " . implode(', ', $categories) . "\n"; | |
| 8067 | - } | |
| 8068 | - | |
| 8069 | - // Get Custom Product Tabs (supports "Custom Product Tabs for WooCommerce" by Code Parrots) | |
| 8070 | - $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true); | |
| 8071 | - if (!empty($custom_tabs) && is_array($custom_tabs)) { | |
| 8072 | - foreach ($custom_tabs as $tab) { | |
| 8073 | - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : ''); | |
| 8074 | - $tab_content = isset($tab['content']) ? $tab['content'] : ''; | |
| 8075 | - | |
| 8076 | - if (!empty($tab_title) && !empty($tab_content)) { | |
| 8077 | - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n"; | |
| 8078 | - } | |
| 8079 | - } | |
| 8080 | - } | |
| 8081 | - | |
| 8082 | - // Also check for reusable/saved tabs applied to this product | |
| 8083 | - $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true); | |
| 8084 | - if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) { | |
| 8085 | - $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array()); | |
| 8086 | - if (!empty($saved_tabs) && is_array($saved_tabs)) { | |
| 8087 | - foreach ($applied_saved_tabs as $saved_tab_id) { | |
| 8088 | - if (isset($saved_tabs[$saved_tab_id])) { | |
| 8089 | - $tab = $saved_tabs[$saved_tab_id]; | |
| 8090 | - $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : ''); | |
| 8091 | - $tab_content = isset($tab['content']) ? $tab['content'] : ''; | |
| 8092 | - | |
| 8093 | - if (!empty($tab_title) && !empty($tab_content)) { | |
| 8094 | - $content .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n"; | |
| 8095 | - } | |
| 8096 | - } | |
| 8097 | - } | |
| 8098 | - } | |
| 8099 | - } | |
| 8100 | - | |
| 8101 | 9194 | return $this->mxchat_sanitize_content_for_api($content); |
| 8102 | 9195 | } |
| 8103 | 9196 | |
| 8104 | 9197 | /** |
| @@ -8128,8 +9221,9 @@ | ||
| 8128 | 9221 | return new WP_Error('page_not_found', 'Page ' . $page_number . ' not found in PDF'); |
| 8129 | 9222 | } |
| 8130 | 9223 | |
| 8131 | 9224 | $text = $pages[$page_number - 1]->getText(); |
| 9225 | + $text = MxChat_Utils::normalize_pdf_rtl($text, 'kb_pdf_page page ' . $page_number); | |
| 8132 | 9226 | |
| 8133 | 9227 | if (empty($text)) { |
| 8134 | 9228 | return new WP_Error('empty_page', 'Page ' . $page_number . ': No text could be extracted — page may contain only images, links, or non-standard encoding'); |
| 8135 | 9229 | } |
| @@ -8150,24 +9244,18 @@ | ||
| 8150 | 9244 | |
| 8151 | 9245 | $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized; |
| 8152 | 9246 | $page_url = esc_url($pdf_url . "#page=" . $page_number); |
| 8153 | 9247 | |
| 8154 | - // Get bot-specific API key | |
| 9248 | + // Get bot-specific embedding decision — custom-provider-aware | |
| 9249 | + // (plan cbd5fd). Error code preserved. | |
| 8155 | 9250 | $bot_options = $this->get_bot_options($bot_id); |
| 8156 | 9251 | $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options'); |
| 8157 | - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002'; | |
| 8158 | - | |
| 8159 | - if (strpos($selected_model, 'voyage') === 0) { | |
| 8160 | - $api_key = $options['voyage_api_key'] ?? ''; | |
| 8161 | - } elseif (strpos($selected_model, 'gemini-embedding') === 0) { | |
| 8162 | - $api_key = $options['gemini_api_key'] ?? ''; | |
| 8163 | - } else { | |
| 8164 | - $api_key = $options['api_key'] ?? ''; | |
| 9252 | + | |
| 9253 | + $preflight = MxChat_Utils::embedding_preflight($options); | |
| 9254 | + if (!$preflight['ok']) { | |
| 9255 | + return new WP_Error('no_api_key', $preflight['reason'] . ' (bot: ' . $bot_id . ')'); | |
| 8165 | 9256 | } |
| 8166 | - | |
| 8167 | - if (empty($api_key)) { | |
| 8168 | - return new WP_Error('no_api_key', 'API key not configured for bot: ' . $bot_id); | |
| 8169 | - } | |
| 9257 | + $api_key = $preflight['api_key']; | |
| 8170 | 9258 | |
| 8171 | 9259 | // Submit to database - UPDATED 2.5.6: Added content_type 'pdf' |
| 8172 | 9260 | $result = MxChat_Utils::submit_content_to_db( |
| 8173 | 9261 | $content_with_metadata, |
