← All changes | includes/class-mxchat-utils.php +93 -1955 3.2.19 → 1.0.8 View file →

		@@ -1,1955 +1,93 @@
1		-<?php
2		-if (!defined('ABSPATH')) {
3		- exit; // Exit if accessed directly
4		-}
5		-
6		-class MxChat_Utils {
7		-
8		-/**
9		- * Validate a client-supplied session id (plan-mxchat-20260731-d42bec).
10		- *
11		- * sanitize_text_field() — which every session_id read site used before this —
12		- * preserves '/' and '..'. Harmless where the value is only an option or
13		- * transient key suffix, but mxchat_send_delayed_transcript() interpolates it
14		- * into a filesystem path, so '../../../../path/x' wrote, emailed and deleted a
15		- * file outside the uploads dir.
16		- *
17		- * REJECTS rather than rewrites: a silently-stripped id would orphan the
18		- * conversation it belongs to, which is harder to diagnose than a clean refusal.
19		- * Returns '' for anything malformed, so call sites fall into the empty-session
20		- * error paths they already have.
21		- *
22		- * The generator only ever emits 'mxchat_chat_' + 32 hex chars
23		- * (class-mxchat-integrator.php, js/chat-script.js), so this is not restrictive
24		- * in practice. Length ceiling is deliberate — session ids are also used as
25		- * option-name suffixes, and WP option names cap at 191 chars.
26		- *
27		- * @param mixed $raw Raw request value.
28		- * @return string The id if well-formed, '' otherwise.
29		- */
30		-public static function sanitize_session_id($raw) {
31		- if (!is_scalar($raw)) {
32		- return '';
33		- }
34		- $val = trim((string) $raw);
35		- if ($val === '') {
36		- return '';
37		- }
38		- return preg_match('/\A[A-Za-z0-9_-]{1,128}\z/', $val) ? $val : '';
39		-}
40		-
41		-/**
42		- * Per-request cache for get_session_history(). Mirrors get_option()'s
43		- * request-scoped caching, which the mxchat_history_ option reads got for
44		- * free before plan 839c4c moved history reads onto the transcripts table.
45		- */
46		-private static $history_cache = array();
47		-
48		-/**
49		- * Session chat history read from the transcripts table, in the exact array
50		- * shape the legacy mxchat_history_<sid> option stored (plan 839c4c). The
51		- * option was a second copy of state the table already held — measured
52		- * byte-identical in role/content/order on 174 of 177 real sessions, with
53		- * the table a superset on the rest — at up to 64 KB per option row. The
54		- * table is now the single store; nothing writes the option any more.
55		- *
56		- * Shape notes, load-bearing for the consumers:
57		- * - id: the transcripts row id (int). Integer ids make the pollers'
58		- * ">" comparisons correct where the old uniqid() strings only worked by
59		- * accident of hex ordering.
60		- * - timestamp: milliseconds, derived from the table's second-resolution GMT
61		- * column (x1000). Consumers comparing against a real-millisecond client
62		- * cutoff MUST floor the cutoff to the second and err inclusive — see the
63		- * persistence-off filters in class-mxchat-integrator.php.
64		- * - agent_name: the row's user_identifier, which the writer sets to the
65		- * same displayed_name value the option carried (agent name when present,
66		- * else email, else identifier).
67		- *
68		- * Public and static so mxchat-woo / mxchat-forms can call the same accessor
69		- * as core, guarded with method_exists against an older mxchat-basic.
70		- *
71		- * @param string $session_id
72		- * @return array[] Chronological entries: id, role, content, timestamp, agent_name.
73		- */
74		-public static function get_session_history($session_id) {
75		- global $wpdb;
76		-
77		- $session_id = self::sanitize_session_id($session_id);
78		- if ($session_id === '') {
79		- return array();
80		- }
81		-
82		- if (array_key_exists($session_id, self::$history_cache)) {
83		- return self::$history_cache[$session_id];
84		- }
85		-
86		- $table = $wpdb->prefix . 'mxchat_chat_transcripts';
87		-
88		- // No SHOW TABLES guard: this is the chat hot path and the table is
89		- // created on activation (with an admin-load safety net). A genuinely
90		- // missing table fails the query and yields the same empty history the
91		- // old option read produced on a fresh session.
92		- $rows = $wpdb->get_results(
93		- $wpdb->prepare(
94		- "SELECT id, role, message, user_identifier, timestamp
95		- FROM `$table` WHERE session_id = %s ORDER BY id ASC",
96		- $session_id
97		- ),
98		- ARRAY_A
99		- );
100		-
101		- $history = array();
102		- if (is_array($rows)) {
103		- foreach ($rows as $row) {
104		- // The column stores GMT (current_time('mysql', 1) at the writer),
105		- // so pin the parse to UTC rather than the site timezone.
106		- $ts = strtotime($row['timestamp'] . ' +0000');
107		- $history[] = array(
108		- 'id' => (int) $row['id'],
109		- 'role' => (string) $row['role'],
110		- 'content' => (string) $row['message'],
111		- 'timestamp' => ($ts ? $ts : 0) * 1000,
112		- 'agent_name' => (string) $row['user_identifier'],
113		- );
114		- }
115		- }
116		-
117		- self::$history_cache[$session_id] = $history;
118		-
119		- return $history;
120		-}
121		-
122		-/**
123		- * Drop the cached history for one session (or all). The writer calls this
124		- * after every insert so a later read in the same request — e.g. the AI
125		- * context build that follows saving the user's message — sees the new row,
126		- * matching the read-your-own-write behavior update_option() gave the old
127		- * option copy.
128		- *
129		- * @param string|null $session_id Null flushes everything (test seam).
130		- */
131		-public static function flush_session_history_cache($session_id = null) {
132		- if ($session_id === null) {
133		- self::$history_cache = array();
134		- return;
135		- }
136		-
137		- unset(self::$history_cache[(string) $session_id]);
138		-}
139		-
140		-/**
141		- * Most recipients the Notification Email field will accept (plan 2f131a).
142		- * A settings field is not a mailing list.
143		- */
144		-const NOTIFICATION_EMAIL_MAX = 5;
145		-
146		-/**
147		- * Parse the Notification Email field into a list of recipients (plan 2f131a).
148		- *
149		- * THE TRAP THIS EXISTS TO CLOSE: sanitize_email() cannot be the validator for
150		- * this field, because its output for the failing input is VALID. WordPress
151		- * strips the separator and the surplus '@' and concatenates the remains:
152		- *
153		- * support@acme.com, sales@acme.com -> support@acme.comsalesacme.com
154		- *
155		- * and is_email() then returns true on that. So every guard in the plugin passed,
156		- * the address was stored, the autosave ticked green, and both the new-session
157		- * notification and the auto-emailed transcript went to a domain that does not
158		- * exist — with no error anywhere. Validating the RAW part BEFORE sanitizing is
159		- * the whole point; reversing those two lines silently restores the bug.
160		- *
161		- * All-or-nothing by design: if any entry is bad the caller must store NOTHING.
162		- * A partial accept — keeping the good addresses and dropping the bad one — is
163		- * the same defect in a new costume, because the owner still believes everyone
164		- * on their list is being notified.
165		- *
166		- * @param mixed $raw Raw field value, exactly as submitted.
167		- * @return array{emails: string[], error: string} Empty emails + empty error
168		- * means the field was empty.
169		- */
170		-public static function parse_notification_emails($raw) {
171		- $out = array('emails' => array(), 'error' => '');
172		-
173		- if (!is_scalar($raw)) {
174		- $out['error'] = __('The notification email could not be read.', 'mxchat');
175		- return $out;
176		- }
177		-
178		- $raw = trim((string) $raw);
179		- if ($raw === '') {
180		- return $out; // genuinely empty — the caller falls back to admin_email
181		- }
182		-
183		- $seen = array();
184		- foreach (preg_split('/[,;]/', $raw) as $part) {
185		- $part = trim($part);
186		- if ($part === '') {
187		- // A trailing or doubled separator carries no address, so skipping it
188		- // cannot silently drop a recipient. This is the ONLY thing tolerated.
189		- continue;
190		- }
191		-
192		- // RAW first. See the note above — order is load-bearing.
193		- $clean = is_email($part) ? sanitize_email($part) : '';
194		- if ($clean === '' || !is_email($clean)) {
195		- return array(
196		- 'emails' => array(),
197		- 'error' => sprintf(
198		- /* translators: %s: the email address the owner typed. */
199		- __('"%s" is not a valid email address, so nothing was saved. Separate multiple addresses with a comma.', 'mxchat'),
200		- esc_html($part)
201		- ),
202		- );
203		- }
204		-
205		- $key = strtolower($clean);
206		- if (isset($seen[$key])) {
207		- continue; // same address twice would simply mail them twice
208		- }
209		- $seen[$key] = true;
210		- $out['emails'][] = $clean;
211		- }
212		-
213		- if (count($out['emails']) > self::NOTIFICATION_EMAIL_MAX) {
214		- return array(
215		- 'emails' => array(),
216		- 'error' => sprintf(
217		- /* translators: %d: maximum number of notification recipients. */
218		- __('Enter at most %d email addresses, separated by commas.', 'mxchat'),
219		- self::NOTIFICATION_EMAIL_MAX
220		- ),
221		- );
222		- }
223		-
224		- return $out;
225		-}
226		-
227		-/**
228		- * The stored recipient list, ready to hand to wp_mail() (plan 2f131a).
229		- *
230		- * Fallback rule, and it is narrow on purpose: an EMPTY field falls back to the
231		- * site admin address, because that is the documented behaviour and an owner who
232		- * never filled the field in still wants their notifications. A field holding
233		- * something unusable does NOT fall back — it sends nowhere, exactly as before
234		- * this plan. Falling back on bad input would mean a typo silently redirects a
235		- * store's transcripts to a different mailbox than the one on screen.
236		- *
237		- * @param array|null $options mxchat_transcripts_options, or null to read it.
238		- * @return string[] Recipients; empty means do not send.
239		- */
240		-public static function notification_recipients($options = null) {
241		- if (!is_array($options)) {
242		- $options = get_option('mxchat_transcripts_options', array());
243		- if (!is_array($options)) {
244		- $options = array();
245		- }
246		- }
247		-
248		- $raw = isset($options['mxchat_notification_email']) ? $options['mxchat_notification_email'] : '';
249		- $raw = is_scalar($raw) ? trim((string) $raw) : '';
250		-
251		- if ($raw === '') {
252		- $admin = get_option('admin_email');
253		- return is_email($admin) ? array($admin) : array();
254		- }
255		-
256		- $parsed = self::parse_notification_emails($raw);
257		- return $parsed['error'] === '' ? $parsed['emails'] : array();
258		-}
259		-
260		-/**
261		- * Centralized embedding model registry. Single source of truth for dimensions
262		- * and provider, so model-switch protection logic doesn't drift across files.
263		- */
264		-public static function embedding_model_registry() {
265		- return array(
266		- 'text-embedding-ada-002' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'Ada 2'),
267		- 'text-embedding-3-small' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'TE3 Small'),
268		- 'text-embedding-3-large' => array('dims' => 3072, 'provider' => 'openai', 'label' => 'TE3 Large'),
269		- 'voyage-3-large' => array('dims' => 2048, 'provider' => 'voyage', 'label' => 'Voyage-3 Large'),
270		- 'gemini-embedding-001' => array('dims' => 1536, 'provider' => 'gemini', 'label' => 'Gemini Embedding'),
271		- );
272		-}
273		-
274		-public static function embedding_model_dimensions($model) {
275		- $registry = self::embedding_model_registry();
276		- return isset($registry[$model]) ? (int) $registry[$model]['dims'] : 0;
277		-}
278		-
279		-public static function embedding_model_label($model) {
280		- if (is_string($model) && strpos($model, 'custom:') === 0) {
281		- /* translators: %s: the embedding model name configured on the custom provider */
282		- return sprintf(__('%s (custom provider)', 'mxchat'), substr($model, 7));
283		- }
284		- $registry = self::embedding_model_registry();
285		- return isset($registry[$model]) ? $registry[$model]['label'] : $model;
286		-}
287		-
288		-/**
289		- * Returns the model that was last used to actually write embeddings into the
290		- * KB. Differs from the user-selected setting once a switch has happened but
291		- * no re-embed has occurred yet — that's the mismatch state we warn about.
292		- */
293		-public static function get_active_embedding_model() {
294		- return get_option('mxchat_active_embedding_model', '');
295		-}
296		-
297		-/**
298		- * Stamp the model that produced the most recent successful embedding. Called
299		- * from generate_embedding() right after the API responds with a valid vector.
300		- */
301		-public static function stamp_active_embedding_model($model) {
302		- if (!empty($model) && $model !== self::get_active_embedding_model()) {
303		- update_option('mxchat_active_embedding_model', $model, false);
304		- }
305		-}
306		-
307		-/**
308		- * The model name the custom-provider embedding path will send, mirroring the
309		- * fallback chain the request itself uses: dedicated custom embedding model,
310		- * else the custom chat model, else 'default'. Single source shared by
311		- * generate_embedding_custom() and the mismatch-warning "selected" side so the
312		- * two can never drift (plan ae02cb).
313		- */
314		-public static function resolve_custom_embedding_model($options) {
315		- if (isset($options['custom_provider_embedding_model']) && trim((string) $options['custom_provider_embedding_model']) !== '') {
316		- return trim((string) $options['custom_provider_embedding_model']);
317		- }
318		- if (isset($options['custom_provider_model']) && trim((string) $options['custom_provider_model']) !== '') {
319		- return trim((string) $options['custom_provider_model']);
320		- }
321		- return 'default';
322		-}
323		-
324		-/**
325		- * The EFFECTIVE selected embedding model — what the next embed will actually
326		- * use. With custom-provider embeddings on this is the custom identity in the
327		- * same 'custom:<model>' form stamp_active_embedding_model() records, not the
328		- * inert standard dropdown value. Mismatch-warning comparisons must read this,
329		- * never $options['embedding_model'] directly — the dropdown cannot be
330		- * deselected, so reading it raw flags every correctly-configured custom setup.
331		- */
332		-public static function get_selected_embedding_model($options = null) {
333		- if (!is_array($options)) {
334		- $options = get_option('mxchat_options', array());
335		- }
336		- if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
337		- return 'custom:' . self::resolve_custom_embedding_model($options);
338		- }
339		- return $options['embedding_model'] ?? '';
340		-}
341		-
342		-/**
343		- * Extract the 11-character YouTube video ID from a URL, or '' if the URL is
344		- * not a single-video YouTube link. Single source of truth for both the KB
345		- * ingestion side and the chat render side — do not duplicate this parsing.
346		- * Channel, playlist, and search URLs deliberately return '' (only a URL that
347		- * identifies one video can be embedded).
348		- */
349		-public static function parse_youtube_id($url) {
350		- if (!is_string($url) || $url === '' || !preg_match('#^https?://#i', $url)) {
351		- return '';
352		- }
353		- $host = strtolower((string) wp_parse_url($url, PHP_URL_HOST));
354		- $host = preg_replace('/^(www|m)\./', '', $host);
355		- $path = (string) wp_parse_url($url, PHP_URL_PATH);
356		- $id = '';
357		- if ($host === 'youtu.be') {
358		- $segments = explode('/', ltrim($path, '/'));
359		- $id = $segments[0] ?? '';
360		- } elseif (in_array($host, array('youtube.com', 'youtube-nocookie.com'), true)) {
361		- if (preg_match('#^/(?:shorts|embed|live|v)/([A-Za-z0-9_-]+)#', $path, $m)) {
362		- $id = $m[1];
363		- } elseif ($path === '/watch') {
364		- parse_str((string) wp_parse_url($url, PHP_URL_QUERY), $query_vars);
365		- $id = isset($query_vars['v']) ? (string) $query_vars['v'] : '';
366		- }
367		- }
368		- $id = preg_replace('/[^A-Za-z0-9_-]/', '', (string) $id);
369		- return (strlen($id) === 11) ? $id : '';
370		-}
371		-
372		-/**
373		- * plan-mxchat-20260813-f52492 — video-card gating.
374		- *
375		- * Two standalone options (NOT mxchat_options — they skip the sanitize/autosave
376		- * traps entirely), read here so the gate in the integrator and the fields on
377		- * Knowledge -> Chunking & Retrieval can never disagree about a default.
378		- *
379		- * Master switch. Default ON: the card is existing behavior, and this is an
380		- * opt-out for owners who never want one, not a new feature to opt into.
381		- */
382		-public static function video_embed_enabled() {
383		- return get_option('mxchat_video_embed_enabled', 'on') === 'on';
384		-}
385		-
386		-/**
387		- * The video card's OWN confidence floor, as a 0-1 cosine — deliberately not
388		- * the site-wide Similarity Threshold (default 35). "Good enough to quote in
389		- * the answer" and "good enough to put a video on screen" are different
390		- * questions: retrieval is allowed to be generous because the model still
391		- * decides what to say, whereas the card is asserted to the visitor with no
392		- * such filter. Stored as an int percentage to match the site-wide slider's
393		- * convention; the default (55) sits above it on purpose.
394		- *
395		- * MXCHAT_VIDEO_EMBED_THRESHOLD_DEFAULT is the single source of that number.
396		- */
397		-public static function video_embed_threshold() {
398		- $stored = get_option('mxchat_video_embed_threshold', null);
399		- $percent = ($stored === null || $stored === '')
400		- ? MXCHAT_VIDEO_EMBED_THRESHOLD_DEFAULT
401		- : (int) $stored;
402		- if ($percent < 0) { $percent = 0; }
403		- if ($percent > 100) { $percent = 100; }
404		- // Cast: PHP evaluates 100/100 to int(1), so an unclamped return type would
405		- // vary with the stored value. Callers compare against a cosine — keep it float.
406		- return (float) $percent / 100;
407		-}
408		-
409		-/**
410		- * UPDATED: Submit or update content (and its embedding) in the database.
411		- * Stores in Pinecone if enabled, otherwise stores in WordPress DB.
412		- *
413		- * @param string $content The content to be embedded.
414		- * @param string $source_url The source URL of the content.
415		- * @param string $api_key The API key used for generating embeddings.
416		- * @param string $vector_id Optional vector ID for Pinecone (if not provided, will use md5 of URL)
417		- * @param string $bot_id The bot ID for multi-bot support
418		- * @param string $content_type The type of content (post, page, pdf, url, manual, product, etc.)
419		- * @return bool|WP_Error True on success, WP_Error on failure
420		- */
421		-public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
422		- global $wpdb;
423		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
424		-
425		- //error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ', Type: ' . $content_type . ')');
426		- //error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
427		-
428		- // Sanitize the source URL
429		- $source_url = esc_url_raw($source_url);
430		-
431		- // Sanitize content_type
432		- $content_type = sanitize_key($content_type);
433		- if (empty($content_type)) {
434		- $content_type = 'content'; // Fallback for backwards compatibility
435		- }
436		-
437		- // Just ensure UTF-8 validity without aggressive escaping
438		- $safe_content = wp_check_invalid_utf8($content);
439		- // Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D)
440		- $safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
441		-
442		- // Check if chunking should be applied
443		- $chunker = MxChat_Chunker::from_settings();
444		- if ($chunker->should_chunk($safe_content)) {
445		- //error_log('[MXCHAT-DB] Content exceeds chunk threshold, using chunked submission');
446		- return self::submit_chunked_content($safe_content, $source_url, $api_key, $bot_id, $content_type, $chunker);
447		- }
448		-
449		- // UPDATED: Generate the embedding using bot-specific configuration
450		- $embedding_vector = self::generate_embedding($content, $api_key, $bot_id);
451		-
452		- if (!is_array($embedding_vector)) {
453		- // Surface the provider's real reason instead of a fixed string (4a7c0a).
454		- $reason = is_wp_error($embedding_vector)
455		- ? $embedding_vector->get_error_message()
456		- : 'Failed to generate embedding for content';
457		- return new WP_Error('embedding_failed', $reason);
458		- }
459		-
460		- //error_log('[MXCHAT-DB] Embedding generated successfully');
461		-
462		- // UPDATED: Check if Pinecone is enabled for this specific bot
463		- if (self::is_pinecone_enabled_for_bot($bot_id)) {
464		- //error_log('[MXCHAT-DB] Pinecone is enabled for bot ' . $bot_id . ' - using Pinecone storage');
465		- // Store in Pinecone only
466		- return self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id, $content_type);
467		- } else {
468		- //error_log('[MXCHAT-DB] Pinecone not enabled for bot ' . $bot_id . ' - using WordPress storage');
469		- // Store in WordPress database only
470		- $embedding_vector_serialized = maybe_serialize($embedding_vector);
471		- return self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type);
472		- }
473		-}
474		-
475		-/**
476		- * UPDATED: Check if Pinecone is enabled and properly configured for a specific bot
477		- */
478		-private static function is_pinecone_enabled_for_bot($bot_id = 'default') {
479		- // For default bot or when multi-bot is not active, use original method
480		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
481		- return self::is_pinecone_enabled();
482		- }
483		-
484		- // Get bot-specific Pinecone configuration
485		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
486		-
487		- if (empty($bot_pinecone_config)) {
488		- // Fallback to default configuration
489		- return self::is_pinecone_enabled();
490		- }
491		-
492		- $enabled_check = !empty($bot_pinecone_config['use_pinecone']) && $bot_pinecone_config['use_pinecone'];
493		- $api_key_check = !empty($bot_pinecone_config['api_key']);
494		- $host_check = !empty($bot_pinecone_config['host']);
495		-
496		- return $enabled_check && $api_key_check && $host_check;
497		-}
498		-
499		-/**
500		- * Check if Pinecone is enabled and properly configured (original method for default bot)
501		- */
502		-private static function is_pinecone_enabled() {
503		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
504		-
505		- if (empty($pinecone_options)) {
506		- return false;
507		- }
508		-
509		- $enabled_check = !empty($pinecone_options['mxchat_use_pinecone']) && $pinecone_options['mxchat_use_pinecone'] !== '0';
510		- $api_key_check = !empty($pinecone_options['mxchat_pinecone_api_key']);
511		- $host_check = !empty($pinecone_options['mxchat_pinecone_host']);
512		-
513		- return $enabled_check && $api_key_check && $host_check;
514		-}
515		-
516		-/**
517		- * UPDATED: Store content in Pinecone only with bot support
518		- */
519		-private static function store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
520		- //error_log('[MXCHAT-PINECONE] ===== Using Pinecone-only storage for bot ' . $bot_id . ' =====');
521		-
522		- // Get bot-specific Pinecone configuration
523		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
524		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
525		- $api_key = $pinecone_options['mxchat_pinecone_api_key'];
526		- $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
527		- $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
528		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
529		- } else {
530		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
531		- if (empty($bot_pinecone_config)) {
532		- // Fallback to default configuration
533		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
534		- $api_key = $pinecone_options['mxchat_pinecone_api_key'];
535		- $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
536		- $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
537		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
538		- } else {
539		- $api_key = $bot_pinecone_config['api_key'];
540		- $environment = ''; // Not used in new Pinecone API
541		- $index_name = ''; // Not used in new Pinecone API
542		- $namespace = $bot_pinecone_config['namespace'] ?? '';
543		- }
544		- }
545		-
546		- $result = self::store_in_pinecone_main(
547		- $embedding_vector,
548		- $content,
549		- $source_url,
550		- $api_key,
551		- $environment,
552		- $index_name,
553		- $vector_id,
554		- $bot_id,
555		- $namespace,
556		- $content_type
557		- );
558		-
559		- if (is_wp_error($result)) {
560		- //error_log('[MXCHAT-PINECONE] Pinecone storage failed for bot ' . $bot_id . ': ' . $result->get_error_message());
561		- return $result;
562		- }
563		-
564		- //error_log('[MXCHAT-PINECONE] Pinecone storage completed successfully for bot ' . $bot_id);
565		- return true;
566		-}
567		-
568		-/**
569		- * Store content in WordPress database with progressive fallback
570		- * UPDATED 2.5.6: Now includes content_type parameter
571		- */
572		-private static function store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type = 'content') {
573		- global $wpdb;
574		-
575		- //error_log('[MXCHAT-DB] ===== Using WordPress-only storage =====');
576		-
577		- // Sanitize content_type
578		- $content_type = sanitize_key($content_type);
579		- if (empty($content_type)) {
580		- $content_type = 'content'; // Fallback for backwards compatibility
581		- }
582		-
583		- // ===== FIXED: Generate unique identifier for manual content =====
584		- $original_source_url = $source_url;
585		- // Check if this is truly manual content (no URL at all) vs a real URL that filter_var rejects
586		- // filter_var(FILTER_VALIDATE_URL) rejects valid URLs with encoded chars, non-ASCII, fragments, etc.
587		- // Use a looser check: if it starts with http(s):// or has a scheme, it's a URL
588		- $has_url_scheme = !empty($source_url) && preg_match('#^https?://#i', $source_url);
589		- // Treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
590		- $is_legacy_mxchat_url = $has_url_scheme && strpos($source_url, 'mxchat.ai') !== false;
591		- $is_manual_content = empty($source_url) || $source_url === '' || !$has_url_scheme || $is_legacy_mxchat_url;
592		-
593		- if ($is_manual_content) {
594		- // Generate unique identifier for manual content to prevent overwrites
595		- $source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
596		- //error_log('[MXCHAT-DB] Generated unique ID for manual content: ' . $source_url);
597		- }
598		-
599		- // Only check for duplicates if we have a valid source URL (not manual content)
600		- $existing_id = null;
601		- if (!$is_manual_content) {
602		- $existing_id = $wpdb->get_var(
603		- $wpdb->prepare(
604		- "SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
605		- $source_url
606		- )
607		- );
608		- //error_log('[MXCHAT-DB] Checked for existing URL, found ID: ' . ($existing_id ?: 'none'));
609		- } else {
610		- //error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)');
611		- }
612		- // ===== END FIX =====
613		-
614		- // Progressive fallback mechanism for problematic content
615		- $attempt = 1;
616		- $max_attempts = 3;
617		- $current_content = $safe_content;
618		- $result = false;
619		-
620		- while ($attempt <= $max_attempts && $result === false) {
621		- try {
622		- if ($existing_id) {
623		- //error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
624		-
625		- // Update the existing row - UPDATED 2.5.6: Added content_type
626		- $result = $wpdb->update(
627		- $table_name,
628		- array(
629		- 'url' => $source_url,
630		- 'article_content' => $current_content,
631		- 'embedding_vector' => $embedding_vector_serialized,
632		- 'source_url' => $source_url,
633		- 'content_type' => $content_type,
634		- 'timestamp' => current_time('mysql'),
635		- ),
636		- array('id' => $existing_id),
637		- array('%s','%s','%s','%s','%s','%s'),
638		- array('%d')
639		- );
640		- } else {
641		- //error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
642		- //error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
643		-
644		- // Insert a new row - UPDATED 2.5.6: Added content_type
645		- $result = $wpdb->insert(
646		- $table_name,
647		- array(
648		- 'url' => $source_url, // Now unique for manual content
649		- 'article_content' => $current_content,
650		- 'embedding_vector' => $embedding_vector_serialized,
651		- 'source_url' => $source_url, // Now unique for manual content
652		- 'content_type' => $content_type,
653		- 'timestamp' => current_time('mysql'),
654		- ),
655		- array('%s','%s','%s','%s','%s','%s')
656		- );
657		- }
658		-
659		- if ($result === false) {
660		- //error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . ')');
661		- //error_log('[MXCHAT-DB] MySQL Error: ' . $wpdb->last_error);
662		- //error_log('[MXCHAT-DB] MySQL Error Number: ' . $wpdb->last_errno);
663		- //error_log('[MXCHAT-DB] Last Query: ' . substr($wpdb->last_query, 0, 500));
664		- //error_log('[MXCHAT-DB] Content length: ' . strlen($current_content) . ' bytes');
665		- //error_log('[MXCHAT-DB] Embedding vector length: ' . strlen($embedding_vector_serialized) . ' bytes');
666		-
667		- // Progressively apply more aggressive sanitization on failure
668		- if ($attempt === 1) {
669		- // First fallback: Use a more aggressive character filter and shorten
670		- $current_content = preg_replace('/[^\p{L}\p{N}\s.,;:!?()-]/u', '', $current_content);
671		- $current_content = substr($current_content, 0, 50000);
672		- } else if ($attempt === 2) {
673		- // Second fallback: Keep only alphanumeric and basic punctuation, shorten further
674		- $current_content = preg_replace('/[^a-zA-Z0-9\s.,;:!?()-]/u', '', $current_content);
675		- $current_content = substr($current_content, 0, 30000);
676		- }
677		-
678		- $attempt++;
679		- }
680		- } catch (Exception $e) {
681		- //error_log('[MXCHAT-DB] Exception during database operation: ' . $e->getMessage());
682		- $attempt++;
683		- }
684		- }
685		-
686		-if ($result === false) {
687		- //error_log('[MXCHAT-DB] All database operation attempts failed');
688		- //error_log('[MXCHAT-DB] Final MySQL Error: ' . $wpdb->last_error);
689		-
690		- $detailed_error = sprintf(
691		- 'Failed to store content in WordPress database after %d attempts. MySQL Error: %s (Error #%d). Content size: %d bytes, Embedding size: %d bytes',
692		- $max_attempts,
693		- $wpdb->last_error,
694		- $wpdb->last_errno,
695		- strlen($current_content),
696		- strlen($embedding_vector_serialized)
697		- );
698		-
699		- return new WP_Error('database_failed', $detailed_error);
700		-}
701		-
702		- //error_log('[MXCHAT-DB] WordPress database operation completed successfully (Attempt ' . ($attempt - 1) . ')');
703		- return true;
704		-}
705		-
706		-/**
707		- * UPDATED: Store content in Pinecone database with bot support
708		- * UPDATED 2.5.6: Now accepts content_type parameter
709		- */
710		-private static function store_in_pinecone_main($embedding_vector, $content, $url, $api_key, $environment, $index_name, $vector_id = null, $bot_id = 'default', $namespace = '', $content_type = 'content') {
711		- //error_log('[MXCHAT-PINECONE-MAIN] ===== Starting Pinecone storage for bot ' . $bot_id . ' =====');
712		-
713		- // ===== UPDATED: Handle manual content with unique vector IDs =====
714		- if ($vector_id) {
715		- // Use provided vector ID
716		- //error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
717		- } elseif (!empty($url) && preg_match('#^https?://#i', $url)) {
718		- // For URLs, use URL-based ID (existing behavior)
719		- $vector_id = md5($url);
720		- //error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
721		- } else {
722		- // For manual content (empty/no URL scheme), generate unique ID
723		- $vector_id = 'manual_' . time() . '_' . substr(md5($content . microtime(true)), 0, 8);
724		- //error_log('[MXCHAT-PINECONE-MAIN] Generated unique vector ID for manual content: ' . $vector_id);
725		- }
726		- // ===== END UPDATE =====
727		-
728		- // Get host from bot-specific config or fallback to default
729		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
730		- $options = get_option('mxchat_pinecone_addon_options');
731		- $host = $options['mxchat_pinecone_host'] ?? '';
732		- } else {
733		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
734		- if (!empty($bot_pinecone_config)) {
735		- $host = $bot_pinecone_config['host'] ?? '';
736		- } else {
737		- $options = get_option('mxchat_pinecone_addon_options');
738		- $host = $options['mxchat_pinecone_host'] ?? '';
739		- }
740		- }
741		-
742		- //error_log('[MXCHAT-PINECONE-MAIN] Host: ' . $host);
743		- //error_log('[MXCHAT-PINECONE-MAIN] API key length: ' . strlen($api_key));
744		- //error_log('[MXCHAT-PINECONE-MAIN] Bot ID: ' . $bot_id);
745		- //error_log('[MXCHAT-PINECONE-MAIN] Namespace: ' . $namespace);
746		-
747		- if (empty($host)) {
748		- //error_log('[MXCHAT-PINECONE-MAIN] ERROR: Host is empty');
749		- return new WP_Error('pinecone_config', 'Pinecone host is not configured. Please set the host in your bot settings.');
750		- }
751		-
752		- // ===== UPDATED 2.5.6: Use passed content_type or determine from URL if not provided =====
753		- // Sanitize content_type
754		- $content_type = sanitize_key($content_type);
755		- if (empty($content_type)) {
756		- // Fallback to old detection logic for backwards compatibility
757		- $is_product = false;
758		- $content_type = 'manual'; // Default for manual content
759		-
760		- if (!empty($url) && preg_match('#^https?://#i', $url)) {
761		- $is_product = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
762		- $content_type = $is_product ? 'product' : 'content';
763		- }
764		- }
765		-
766		- //error_log('[MXCHAT-PINECONE-MAIN] Content type: ' . $content_type);
767		- // ===== END UPDATE =====
768		-
769		- $api_endpoint = "https://{$host}/vectors/upsert";
770		- //error_log('[MXCHAT-PINECONE-MAIN] API endpoint: ' . $api_endpoint);
771		-
772		- // UPDATED 2.5.6: Use provided content_type in metadata
773		- $metadata = array(
774		- 'text' => $content,
775		- 'source_url' => $url, // Can be empty for manual content
776		- 'type' => $content_type, // Now supports: post, page, pdf, url, manual, product, etc.
777		- 'last_updated' => time(),
778		- 'created_at' => time(), // Add creation timestamp
779		- 'bot_id' => $bot_id, // Add bot identification
780		- );
781		-
782		- $vector_data = array(
783		- 'id' => $vector_id,
784		- 'values' => $embedding_vector,
785		- 'metadata' => $metadata
786		- );
787		-
788		- $request_body = array(
789		- 'vectors' => array($vector_data)
790		- );
791		-
792		- // Add namespace if specified for multi-bot separation
793		- if (!empty($namespace)) {
794		- $request_body['namespace'] = $namespace;
795		- //error_log('[MXCHAT-PINECONE-MAIN] Using namespace: ' . $namespace);
796		- }
797		-
798		- //error_log('[MXCHAT-PINECONE-MAIN] Request body prepared (embedding dimensions: ' . count($embedding_vector) . ')');
799		-
800		- $response = wp_remote_post($api_endpoint, array(
801		- 'headers' => array(
802		- 'Api-Key' => $api_key,
803		- 'accept' => 'application/json',
804		- 'content-type' => 'application/json'
805		- ),
806		- 'body' => wp_json_encode($request_body),
807		- 'timeout' => 30,
808		- 'data_format' => 'body'
809		- ));
810		-
811		- if (is_wp_error($response)) {
812		- //error_log('[MXCHAT-PINECONE-MAIN] WordPress request error: ' . $response->get_error_message());
813		- return new WP_Error('pinecone_request', $response->get_error_message());
814		- }
815		-
816		- $response_code = wp_remote_retrieve_response_code($response);
817		- //error_log('[MXCHAT-PINECONE-MAIN] Response code: ' . $response_code);
818		-
819		- if ($response_code !== 200) {
820		- $body = wp_remote_retrieve_body($response);
821		- //error_log('[MXCHAT-PINECONE-MAIN] API error - Response body: ' . $body);
822		- return new WP_Error('pinecone_api', sprintf(
823		- 'Pinecone API error (HTTP %d): %s',
824		- $response_code,
825		- $body
826		- ));
827		- }
828		-
829		- $response_body = wp_remote_retrieve_body($response);
830		- //error_log('[MXCHAT-PINECONE-MAIN] Success response: ' . $response_body);
831		- //error_log('[MXCHAT-PINECONE-MAIN] Successfully stored in Pinecone for bot ' . $bot_id);
832		- //error_log('[MXCHAT-PINECONE-MAIN] ===== Pinecone storage complete =====');
833		-
834		- return true;
835		-}
836		-
837		-/**
838		- * Caller-side pre-flight for KB ingestion: can an embedding request be made
839		- * with these options, and which API key should travel downstream?
840		- *
841		- * Custom-provider-aware — generate_embedding() below routes to the custom
842		- * endpoint FIRST and ignores the passed cloud key entirely when
843		- * custom_provider_for_embeddings is on, so on that branch the only real
844		- * requirement is a Base URL. Ingestion callers that gated on a cloud API key
845		- * were killing keyless custom-embeddings sites (local Ollama / LM Studio
846		- * class) before the embed layer could route (plan cbd5fd).
847		- *
848		- * NOTE: reads $options['embedding_model'] raw on purpose — this mirrors
849		- * generate_embedding()'s own routing read, NOT the mismatch-banner's
850		- * "selected" chain (get_selected_embedding_model). The helper must predict
851		- * what the very next embed call will do, byte-for-byte.
852		- *
853		- * Decision only — callers keep their own error-surfacing shape (admin-notice
854		- * transient + redirect, wp_send_json_error, WP_Error, silent return).
855		- *
856		- * @param array|null $options Resolved options (bot-specific where the caller
857		- * has them); null loads the default bot's options.
858		- * @return array {
859		- * @type bool $ok Whether ingestion can proceed.
860		- * @type string $api_key Key to pass downstream ('' on the custom branch —
861		- * generate_embedding() ignores it there).
862		- * @type string $reason Human-readable blocker; '' when $ok.
863		- * @type string $provider Short provider label ('OpenAI', 'Voyage AI',
864		- * 'Google Gemini', 'Custom Provider').
865		- * }
866		- */
867		-public static function embedding_preflight($options = null) {
868		- if (!is_array($options)) {
869		- $options = get_option('mxchat_options');
870		- $options = is_array($options) ? $options : array();
871		- }
872		-
873		- // Custom branch mirrors generate_embedding()'s routing order (custom first).
874		- if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
875		- $base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
876		- if ($base_url === '') {
877		- return array(
878		- 'ok' => false,
879		- 'api_key' => '',
880		- // Same string generate_embedding_custom() returns for this state.
881		- 'reason' => __('Custom provider Base URL is not configured.', 'mxchat'),
882		- 'provider' => 'Custom Provider',
883		- );
884		- }
885		- return array('ok' => true, 'api_key' => '', 'reason' => '', 'provider' => 'Custom Provider');
886		- }
887		-
888		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
889		- if (strpos($selected_model, 'voyage') === 0) {
890		- $api_key = $options['voyage_api_key'] ?? '';
891		- $provider = 'Voyage AI';
892		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
893		- $api_key = $options['gemini_api_key'] ?? '';
894		- $provider = 'Google Gemini';
895		- } else {
896		- $api_key = $options['api_key'] ?? '';
897		- $provider = 'OpenAI';
898		- }
899		-
900		- if (empty($api_key)) {
901		- return array(
902		- 'ok' => false,
903		- 'api_key' => '',
904		- 'reason' => sprintf(
905		- /* translators: %s: embedding provider name */
906		- __('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
907		- $provider
908		- ),
909		- 'provider' => $provider,
910		- );
911		- }
912		-
913		- return array('ok' => true, 'api_key' => $api_key, 'reason' => '', 'provider' => $provider);
914		-}
915		-
916		-/**
917		- * Public QUERY-side entry point (plan 876edb). The chat pipeline's
918		- * MxChat_Integrator::mxchat_generate_embedding() adapter routes through here
919		- * so the query and index sides share ONE provider-routing implementation —
920		- * the same endpoints, request bodies, and stamping semantics. The Integrator
921		- * keeps its own error vocabulary by translating the WP_Error this returns
922		- * (see the structured error data on every failure path below).
923		- *
924		- * @param string $text The text to be embedded.
925		- * @param string $api_key Caller-resolved API key (per-bot on the query side).
926		- * @param string $bot_id The bot ID for multi-bot support.
927		- * @return array|WP_Error The embedding vector, or WP_Error carrying the reason.
928		- */
929		-public static function generate_query_embedding($text, $api_key, $bot_id = 'default') {
930		- return self::generate_embedding($text, $api_key, $bot_id);
931		-}
932		-
933		-/**
934		- * UPDATED: Generate an embedding for the given text using bot-specific configuration.
935		- *
936		- * @param string $text The text to be embedded.
937		- * @param string $api_key The API key used for generating embeddings.
938		- * @param string $bot_id The bot ID for multi-bot support
939		- * @return array|null The embedding vector or null on failure.
940		- */
941		-private static function generate_embedding($text, $api_key, $bot_id = 'default') {
942		- // Get bot-specific options
943		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
944		- $options = get_option('mxchat_options');
945		- } else {
946		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
947		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
948		- }
949		-
950		- // Opt-in: when the custom provider is selected for embeddings, route the KB
951		- // INDEX side through the same custom endpoint the query side uses, so stored
952		- // vectors and query vectors come from the same model. Default-off behavior
953		- // below is untouched.
954		- if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
955		- $custom = self::generate_embedding_custom($text, $options);
956		- // The custom path already returns a human-readable error string —
957		- // carry it instead of collapsing to null (plan 4a7c0a). The 'custom'
958		- // branch marker lets the Integrator adapter map the string back onto
959		- // its own error codes (876edb).
960		- return is_array($custom) ? $custom : new WP_Error('embedding_failed', (string) $custom, array('branch' => 'custom'));
961		- }
962		-
963		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
964		-
965		- // Determine endpoint and API key based on model
966		- if (strpos($selected_model, 'voyage') === 0) {
967		- $endpoint = 'https://api.voyageai.com/v1/embeddings';
968		- $api_key = $options['voyage_api_key'] ?? '';
969		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
970		- $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
971		- $api_key = $options['gemini_api_key'] ?? '';
972		- } else {
973		- $endpoint = 'https://api.openai.com/v1/embeddings';
974		- // Prefer the caller-resolved key when one was passed — the query side
975		- // resolves per-bot keys at its call sites (integrator adapter, 876edb).
976		- // Index callers pass the preflight key, which equals this options read,
977		- // so nothing changes for them.
978		- $api_key = !empty($api_key) ? $api_key : ($options['api_key'] ?? '');
979		- }
980		-
981		- // Prepare request body based on provider
982		- if (strpos($selected_model, 'gemini-embedding') === 0) {
983		- // Gemini API format
984		- $request_body = [
985		- 'model' => 'models/' . $selected_model,
986		- 'content' => [
987		- 'parts' => [
988		- ['text' => $text]
989		- ]
990		- ],
991		- 'outputDimensionality' => 1536
992		- ];
993		-
994		- // Prepare headers for Gemini (API key as query parameter)
995		- $endpoint .= '?key=' . $api_key;
996		- $headers = [
997		- 'Content-Type' => 'application/json'
998		- ];
999		- } else {
1000		- // OpenAI/Voyage API format
1001		- $request_body = [
1002		- 'input' => $text,
1003		- 'model' => $selected_model
1004		- ];
1005		-
1006		- // Add output_dimension for voyage-3-large
1007		- if ($selected_model === 'voyage-3-large') {
1008		- $request_body['output_dimension'] = 2048;
1009		- }
1010		-
1011		- // Prepare headers for OpenAI/Voyage
1012		- $headers = [
1013		- 'Content-Type' => 'application/json',
1014		- 'Authorization' => 'Bearer ' . $api_key
1015		- ];
1016		- }
1017		-
1018		- $args = [
1019		- 'body' => wp_json_encode($request_body),
1020		- 'headers' => $headers,
1021		- 'timeout' => 60,
1022		- 'redirection' => 5,
1023		- 'blocking' => true,
1024		- 'httpversion' => '1.0',
1025		- 'sslverify' => true,
1026		- ];
1027		-
1028		- $response = wp_remote_post($endpoint, $args);
1029		-
1030		- if (is_wp_error($response)) {
1031		- $message = 'Embedding request failed (connection): ' . $response->get_error_message();
1032		- if (class_exists('MxChat_Admin')) {
1033		- MxChat_Admin::mxchat_log_debug('embedding_error', $message, array('model' => $selected_model, 'bot_id' => $bot_id));
1034		- }
1035		- return new WP_Error('embedding_failed', $message, array(
1036		- 'branch' => 'cloud',
1037		- 'kind' => 'connection',
1038		- 'reason' => $response->get_error_message(),
1039		- 'model' => $selected_model,
1040		- ));
1041		- }
1042		-
1043		- $response_body = json_decode(wp_remote_retrieve_body($response), true);
1044		-
1045		- // Handle different response formats based on provider
1046		- if (strpos($selected_model, 'gemini-embedding') === 0) {
1047		- // Gemini API response format
1048		- if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
1049		- self::stamp_active_embedding_model($selected_model);
1050		- return $response_body['embedding']['values'];
1051		- } else {
1052		- return self::embedding_failure_error($response, $selected_model, $api_key, $bot_id);
1053		- }
1054		- } else {
1055		- // OpenAI/Voyage API response format
1056		- if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
1057		- self::stamp_active_embedding_model($selected_model);
1058		- return $response_body['data'][0]['embedding'];
1059		- } else {
1060		- return self::embedding_failure_error($response, $selected_model, $api_key, $bot_id);
1061		- }
1062		- }
1063		-}
1064		-
1065		-/**
1066		- * Build a WP_Error carrying the embedding provider's REAL failure reason,
1067		- * and record it in the Debug Mode log. Previously every failure path
1068		- * returned bare null, so customers saw only "Failed to generate embedding
1069		- * for content" / "Failed to store any chunks" with no cause (plan 4a7c0a).
1070		- *
1071		- * The API key never appears in provider response bodies (it travels in the
1072		- * request headers), but the reason is scrubbed for it anyway before it can
1073		- * reach a notice or the debug log.
1074		- */
1075		-private static function embedding_failure_error($response, $selected_model, $api_key, $bot_id) {
1076		- $status = (int) wp_remote_retrieve_response_code($response);
1077		- $raw = (string) wp_remote_retrieve_body($response);
1078		- $decoded = json_decode($raw, true);
1079		-
1080		- // Provider error shapes: OpenAI + Gemini use {"error":{"message":…}};
1081		- // Voyage uses {"detail":…}.
1082		- $reason = '';
1083		- if (is_array($decoded)) {
1084		- if (isset($decoded['error']['message']) && is_string($decoded['error']['message'])) {
1085		- $reason = $decoded['error']['message'];
1086		- } elseif (isset($decoded['detail']) && is_string($decoded['detail'])) {
1087		- $reason = $decoded['detail'];
1088		- }
1089		- }
1090		- if ($reason === '') {
1091		- $reason = ($raw !== '') ? substr($raw, 0, 200) : 'empty or malformed response';
1092		- }
1093		- if (is_string($api_key) && $api_key !== '') {
1094		- $reason = str_replace($api_key, '[redacted]', $reason);
1095		- }
1096		- $reason = substr($reason, 0, 300);
1097		- $message = sprintf('Embedding failed (%s, HTTP %d): %s', $selected_model, $status, $reason);
1098		-
1099		- if (class_exists('MxChat_Admin')) {
1100		- MxChat_Admin::mxchat_log_debug('embedding_error', $message, array(
1101		- 'model' => $selected_model,
1102		- 'status' => $status,
1103		- 'bot_id' => $bot_id,
1104		- ));
1105		- }
1106		-
1107		- // Structured data so the Integrator's query-side adapter can rebuild its
1108		- // typed error contract (auth/rate-limit/quota/invalid-response) without a
1109		- // second transport implementation (876edb). Additive — message unchanged.
1110		- return new WP_Error('embedding_failed', $message, array(
1111		- 'branch' => 'cloud',
1112		- 'status' => $status,
1113		- 'error_type' => (is_array($decoded) && isset($decoded['error']['type']) && is_string($decoded['error']['type'])) ? $decoded['error']['type'] : '',
1114		- 'reason' => $reason,
1115		- 'model' => $selected_model,
1116		- ));
1117		-}
1118		-
1119		-/**
1120		- * Generate an embedding via a Custom (OpenAI-compatible) provider's /embeddings route.
1121		- * Shared by every embedding entry point so the KNOWLEDGE-BASE INDEX side and the
1122		- * QUERY side route through the same model when the opt-in
1123		- * 'custom_provider_for_embeddings' setting is on. Mirrors the query-path logic in
1124		- * MxChat_Integrator::mxchat_generate_embedding_custom() but takes an explicit
1125		- * $options array so it is callable statically from utils + knowledge-manager.
1126		- *
1127		- * Returns a numeric array (the embedding vector) on success, or a human-readable
1128		- * error string on failure (so callers expecting a string error, like the
1129		- * knowledge-manager, can surface it directly; callers expecting array|null wrap it).
1130		- *
1131		- * @param string $text Text to embed.
1132		- * @param array $options The resolved mxchat options (must contain the custom_provider_* keys).
1133		- * @return array|string Embedding vector on success; error string on failure.
1134		- */
1135		-public static function generate_embedding_custom($text, $options) {
1136		- if (empty($text)) {
1137		- return 'No text provided for embedding generation';
1138		- }
1139		-
1140		- $base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
1141		- if (empty($base_url)) {
1142		- return 'Custom provider Base URL is not configured.';
1143		- }
1144		-
1145		- $api_key = isset($options['custom_provider_api_key']) ? trim((string) $options['custom_provider_api_key']) : '';
1146		- $auth_scheme = isset($options['custom_provider_auth_scheme']) ? $options['custom_provider_auth_scheme'] : 'bearer';
1147		- $api_version = isset($options['custom_provider_api_version']) ? trim((string) $options['custom_provider_api_version']) : '';
1148		-
1149		- // Embedding model: shared resolver (dedicated embedding model -> chat model
1150		- // -> 'default') — the mismatch warning's "selected" side reads the same chain.
1151		- $model = self::resolve_custom_embedding_model($options);
1152		-
1153		- $embed_url = $base_url . '/embeddings';
1154		- if (!empty($api_version)) {
1155		- $embed_url .= (strpos($embed_url, '?') === false ? '?' : '&') . 'api-version=' . rawurlencode($api_version);
1156		- }
1157		-
1158		- $headers = ['Content-Type' => 'application/json'];
1159		- if (!empty($api_key)) {
1160		- if ($auth_scheme === 'api-key') {
1161		- $headers['api-key'] = $api_key;
1162		- } else {
1163		- $headers['Authorization'] = 'Bearer ' . $api_key;
1164		- }
1165		- }
1166		-
1167		- $response = wp_remote_post($embed_url, [
1168		- 'headers' => $headers,
1169		- 'body' => wp_json_encode(['input' => $text, 'model' => $model]),
1170		- 'timeout' => 60,
1171		- ]);
1172		- if (is_wp_error($response)) {
1173		- return self::log_custom_embedding_failure(
1174		- 'Connection error when generating embeddings (custom provider): ' . $response->get_error_message(),
1175		- $model,
1176		- $api_key
1177		- );
1178		- }
1179		-
1180		- $status = wp_remote_retrieve_response_code($response);
1181		- $body = json_decode(wp_remote_retrieve_body($response), true);
1182		- if ($status !== 200) {
1183		- $msg = isset($body['error']['message']) ? $body['error']['message'] : 'HTTP ' . $status;
1184		- return self::log_custom_embedding_failure(
1185		- 'Custom embedding endpoint error: ' . $msg,
1186		- $model,
1187		- $api_key,
1188		- (int) $status
1189		- );
1190		- }
1191		- if (isset($body['data'][0]['embedding']) && is_array($body['data'][0]['embedding'])) {
1192		- // Stamp the custom model identity so the active-embedding-model mismatch
1193		- // warning reflects the real (custom) model rather than the built-in setting.
1194		- self::stamp_active_embedding_model('custom:' . $model);
1195		- return $body['data'][0]['embedding'];
1196		- }
1197		- return self::log_custom_embedding_failure('Invalid embedding response from custom provider.', $model, $api_key);
1198		-}
1199		-
1200		-/**
1201		- * Record a custom-provider embedding failure in the Debug Mode log, then
1202		- * return the message unchanged so callers keep their string-error contract.
1203		- * The cloud branch has logged its failures since 4a7c0a; the custom branch
1204		- * never did, so chat-side failures on Custom-provider installs were
1205		- * invisible to Debug Mode despite the 3.2.18 readme saying otherwise
1206		- * (plan 71e4b6). Same scrub-then-log shape as embedding_failure_error().
1207		- *
1208		- * @param string $message Human-readable failure (the caller's return value).
1209		- * @param string $model Resolved custom embedding model.
1210		- * @param string $api_key Scrubbed out of the logged message if it ever appears.
1211		- * @param int $status HTTP status when one was received, 0 otherwise.
1212		- * @return string The (scrubbed) message.
1213		- */
1214		-private static function log_custom_embedding_failure($message, $model, $api_key, $status = 0) {
1215		- if (is_string($api_key) && $api_key !== '') {
1216		- $message = str_replace($api_key, '[redacted]', $message);
1217		- }
1218		-
1219		- if (class_exists('MxChat_Admin')) {
1220		- $context = array('model' => 'custom:' . $model);
1221		- if ($status > 0) {
1222		- $context['status'] = $status;
1223		- }
1224		- MxChat_Admin::mxchat_log_debug('embedding_error', $message, $context);
1225		- }
1226		-
1227		- return $message;
1228		-}
1229		-
1230		-/**
1231		- * Submit content as multiple chunks
1232		- *
1233		- * Splits large content into chunks, generates embeddings for each,
1234		- * and stores them with chunk metadata for later reassembly.
1235		- *
1236		- * @param string $content The content to chunk and store
1237		- * @param string $source_url The source URL
1238		- * @param string $api_key The API key for embeddings
1239		- * @param string $bot_id The bot ID
1240		- * @param string $content_type The content type
1241		- * @param MxChat_Chunker $chunker The chunker instance
1242		- * @return bool|WP_Error True on success, WP_Error on failure
1243		- */
1244		-private static function submit_chunked_content($content, $source_url, $api_key, $bot_id, $content_type, $chunker) {
1245		- global $wpdb;
1246		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1247		-
1248		- //error_log('[MXCHAT-CHUNK-DEBUG] Starting chunked submission for: ' . $source_url);
1249		- //error_log('[MXCHAT-CHUNK-DEBUG] Content length: ' . strlen($content) . ' chars');
1250		-
1251		- // First, delete any existing chunks for this URL (clean slate)
1252		- $delete_result = self::delete_chunks_for_url($source_url, $bot_id);
1253		- if (is_wp_error($delete_result)) {
1254		- //error_log('[MXCHAT-CHUNK-DEBUG] Warning: Failed to delete existing chunks: ' . $delete_result->get_error_message());
1255		- // Continue anyway - we'll overwrite with upsert
1256		- }
1257		-
1258		- // Split content into chunks
1259		- $chunks = $chunker->chunk_text($content);
1260		- $total_chunks = count($chunks);
1261		-
1262		- //error_log('[MXCHAT-CHUNK-DEBUG] Created ' . $total_chunks . ' chunks');
1263		- foreach ($chunks as $i => $chunk) {
1264		- //error_log('[MXCHAT-CHUNK-DEBUG] Chunk ' . $i . ' length: ' . strlen($chunk) . ' chars, preview: ' . substr($chunk, 0, 100));
1265		- }
1266		-
1267		- //error_log('[MXCHAT-CHUNK] Split content into ' . $total_chunks . ' chunks');
1268		-
1269		- if ($total_chunks === 0) {
1270		- return new WP_Error('chunking_failed', 'Content could not be split into chunks');
1271		- }
1272		-
1273		- $errors = array();
1274		- $embed_failures = 0;
1275		- $first_embed_reason = '';
1276		- $first_store_reason = '';
1277		- $is_pinecone = self::is_pinecone_enabled_for_bot($bot_id);
1278		-
1279		- foreach ($chunks as $index => $chunk_text) {
1280		- // Generate chunk metadata
1281		- $chunk_metadata = MxChat_Chunker::create_chunk_metadata($index, $total_chunks, $source_url);
1282		-
1283		- // AI-Engine-style aliases so external consumers (Pinecone/Qdrant/Chroma) can rely on
1284		- // a stable shorthand ('source'/'part_index'/'part_total') without parsing our internal names.
1285		- $chunk_metadata['source'] = $source_url;
1286		- $chunk_metadata['part_index'] = (int) $index;
1287		- $chunk_metadata['part_total'] = (int) $total_chunks;
1288		-
1289		- /**
1290		- * Filter the per-chunk metadata blob before it's written to the KB store.
1291		- *
1292		- * @param array $chunk_metadata Metadata array (source, part_index, part_total, chunk_index, total_chunks, source_url, parent_url_hash, document_type, ...).
1293		- * @param string $chunk_text The chunk text being stored.
1294		- * @param array $context ['bot_id' => string, 'content_type' => string, 'source_url' => string, 'part_index' => int, 'part_total' => int]
1295		- * @return array Updated metadata array.
1296		- */
1297		- $chunk_metadata = apply_filters(
1298		- 'mxchat_embedding_chunk_metadata',
1299		- $chunk_metadata,
1300		- $chunk_text,
1301		- array(
1302		- 'bot_id' => $bot_id,
1303		- 'content_type' => $content_type,
1304		- 'source_url' => $source_url,
1305		- 'part_index' => (int) $index,
1306		- 'part_total' => (int) $total_chunks,
1307		- )
1308		- );
1309		-
1310		- $chunk_vector_id = MxChat_Chunker::generate_chunk_vector_id($source_url, $index);
1311		-
1312		- //error_log('[MXCHAT-CHUNK] Processing chunk ' . ($index + 1) . '/' . $total_chunks . ' (ID: ' . $chunk_vector_id . ')');
1313		-
1314		- // Generate embedding for this chunk
1315		- $embedding_vector = self::generate_embedding($chunk_text, $api_key, $bot_id);
1316		-
1317		- if (!is_array($embedding_vector)) {
1318		- // Track embedding failures separately from storage failures, and
1319		- // keep the first provider reason seen — the two failure classes
1320		- // have opposite remedies (API key vs Pinecone/DB) (plan 4a7c0a).
1321		- $embed_failures++;
1322		- $reason = is_wp_error($embedding_vector) ? $embedding_vector->get_error_message() : '';
1323		- if ($reason !== '' && $first_embed_reason === '') {
1324		- $first_embed_reason = $reason;
1325		- }
1326		- $errors[] = new WP_Error('embedding_failed', 'Failed to generate embedding for chunk ' . $index . ($reason !== '' ? ' — ' . $reason : ''));
1327		- continue;
1328		- }
1329		-
1330		- if ($is_pinecone) {
1331		- // Store in Pinecone with chunk metadata
1332		- $result = self::store_chunk_in_pinecone(
1333		- $embedding_vector,
1334		- $chunk_text,
1335		- $source_url,
1336		- $chunk_vector_id,
1337		- $bot_id,
1338		- $content_type,
1339		- $chunk_metadata
1340		- );
1341		- } else {
1342		- // Store in WordPress DB with chunk metadata
1343		- $content_with_metadata = MxChat_Chunker::format_chunk_for_storage($chunk_text, $chunk_metadata);
1344		- $embedding_vector_serialized = maybe_serialize($embedding_vector);
1345		-
1346		- $result = self::store_chunk_in_wordpress_db(
1347		- $content_with_metadata,
1348		- $source_url,
1349		- $embedding_vector_serialized,
1350		- $table_name,
1351		- $content_type,
1352		- $chunk_metadata
1353		- );
1354		- }
1355		-
1356		- if (is_wp_error($result)) {
1357		- $errors[] = $result;
1358		- if ($first_store_reason === '') {
1359		- $first_store_reason = $result->get_error_message();
1360		- }
1361		- }
1362		- }
1363		-
1364		- if (count($errors) === $total_chunks) {
1365		- // Say WHICH stage failed — "failed to store" used to cover pure
1366		- // embedding failures too, sending customers to debug Pinecone when
1367		- // the problem was their embedding API key (plan 4a7c0a).
1368		- if ($embed_failures === $total_chunks) {
1369		- return new WP_Error('chunking_failed',
1370		- 'Failed to store any chunks — every chunk failed to embed'
1371		- . ($first_embed_reason !== '' ? ': ' . $first_embed_reason : '')
1372		- . ' Check the embedding provider API key and model under MxChat Settings.');
1373		- }
1374		- if ($embed_failures === 0) {
1375		- return new WP_Error('chunking_failed',
1376		- 'Failed to store any chunks — embeddings generated but storage failed'
1377		- . ($first_store_reason !== '' ? ': ' . $first_store_reason : '')
1378		- . ' Check the knowledge base storage (Pinecone index or database).');
1379		- }
1380		- return new WP_Error('chunking_failed', sprintf(
1381		- 'Failed to store any chunks — %d failed to embed%s and %d failed to store%s',
1382		- $embed_failures,
1383		- $first_embed_reason !== '' ? ' (' . $first_embed_reason . ')' : '',
1384		- $total_chunks - $embed_failures,
1385		- $first_store_reason !== '' ? ' (' . $first_store_reason . ')' : ''
1386		- ));
1387		- }
1388		-
1389		- if (!empty($errors)) {
1390		- $detail = $first_embed_reason !== '' ? $first_embed_reason : $first_store_reason;
1391		- return new WP_Error('chunking_partial_failure',
1392		- sprintf('Failed to store %d of %d chunks', count($errors), $total_chunks)
1393		- . ($detail !== '' ? ' — first error: ' . $detail : ''));
1394		- }
1395		-
1396		- //error_log('[MXCHAT-CHUNK] Successfully stored all ' . $total_chunks . ' chunks');
1397		- return true;
1398		-}
1399		-
1400		-/**
1401		- * Store a single chunk in Pinecone with chunk-specific metadata
1402		- */
1403		-private static function store_chunk_in_pinecone($embedding_vector, $chunk_text, $source_url, $vector_id, $bot_id, $content_type, $chunk_metadata) {
1404		- // Get Pinecone configuration
1405		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
1406		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
1407		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1408		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1409		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1410		- } else {
1411		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1412		- if (empty($bot_pinecone_config)) {
1413		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
1414		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1415		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1416		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1417		- } else {
1418		- $api_key = $bot_pinecone_config['api_key'] ?? '';
1419		- $host = $bot_pinecone_config['host'] ?? '';
1420		- $namespace = $bot_pinecone_config['namespace'] ?? '';
1421		- }
1422		- }
1423		-
1424		- if (empty($host) || empty($api_key)) {
1425		- return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
1426		- }
1427		-
1428		- $api_endpoint = "https://{$host}/vectors/upsert";
1429		-
1430		- // Build metadata with chunk information
1431		- $metadata = array(
1432		- 'text' => $chunk_text,
1433		- 'source_url' => $source_url,
1434		- 'type' => $content_type,
1435		- 'is_chunked' => true,
1436		- 'chunk_index' => $chunk_metadata['chunk_index'],
1437		- 'total_chunks' => $chunk_metadata['total_chunks'],
1438		- 'parent_url_hash' => $chunk_metadata['parent_url_hash'],
1439		- 'last_updated' => time(),
1440		- 'created_at' => time(),
1441		- 'bot_id' => $bot_id,
1442		- );
1443		-
1444		- $vector_data = array(
1445		- 'id' => $vector_id,
1446		- 'values' => $embedding_vector,
1447		- 'metadata' => $metadata
1448		- );
1449		-
1450		- $request_body = array(
1451		- 'vectors' => array($vector_data)
1452		- );
1453		-
1454		- if (!empty($namespace)) {
1455		- $request_body['namespace'] = $namespace;
1456		- }
1457		-
1458		- $response = wp_remote_post($api_endpoint, array(
1459		- 'headers' => array(
1460		- 'Api-Key' => $api_key,
1461		- 'accept' => 'application/json',
1462		- 'content-type' => 'application/json'
1463		- ),
1464		- 'body' => wp_json_encode($request_body),
1465		- 'timeout' => 30
1466		- ));
1467		-
1468		- if (is_wp_error($response)) {
1469		- return $response;
1470		- }
1471		-
1472		- $response_code = wp_remote_retrieve_response_code($response);
1473		- if ($response_code !== 200) {
1474		- return new WP_Error('pinecone_api', 'Pinecone API error: HTTP ' . $response_code);
1475		- }
1476		-
1477		- return true;
1478		-}
1479		-
1480		-/**
1481		- * Store a single chunk in WordPress database
1482		- */
1483		-private static function store_chunk_in_wordpress_db($content_with_metadata, $source_url, $embedding_vector_serialized, $table_name, $content_type, $chunk_metadata) {
1484		- global $wpdb;
1485		-
1486		- // For chunks, we always insert new rows (no duplicate checking)
1487		- // The URL includes chunk info in the metadata, but source_url stays the same for grouping
1488		- $result = $wpdb->insert(
1489		- $table_name,
1490		- array(
1491		- 'url' => $source_url,
1492		- 'article_content' => $content_with_metadata,
1493		- 'embedding_vector' => $embedding_vector_serialized,
1494		- 'source_url' => $source_url,
1495		- 'content_type' => $content_type,
1496		- 'timestamp' => current_time('mysql')
1497		- ),
1498		- array('%s', '%s', '%s', '%s', '%s', '%s')
1499		- );
1500		-
1501		- if ($result === false) {
1502		- return new WP_Error('database_failed', 'Failed to insert chunk: ' . $wpdb->last_error);
1503		- }
1504		-
1505		- return true;
1506		-}
1507		-
1508		-/**
1509		- * Delete all chunks for a given URL
1510		- *
1511		- * @param string $source_url The source URL
1512		- * @param string $bot_id The bot ID
1513		- * @return bool|WP_Error True on success, WP_Error on failure
1514		- */
1515		-public static function delete_chunks_for_url($source_url, $bot_id = 'default') {
1516		- //error_log('[MXCHAT-CHUNK-DELETE] Deleting chunks for URL: ' . $source_url);
1517		-
1518		- if (self::is_pinecone_enabled_for_bot($bot_id)) {
1519		- return self::delete_pinecone_chunks_by_url($source_url, $bot_id);
1520		- } else {
1521		- return self::delete_wordpress_chunks_by_url($source_url);
1522		- }
1523		-}
1524		-
1525		-/**
1526		- * Delete all chunks for a URL from Pinecone
1527		- */
1528		-private static function delete_pinecone_chunks_by_url($source_url, $bot_id) {
1529		- // Get Pinecone configuration
1530		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
1531		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
1532		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1533		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1534		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1535		- } else {
1536		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1537		- if (empty($bot_pinecone_config)) {
1538		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
1539		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1540		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1541		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1542		- } else {
1543		- $api_key = $bot_pinecone_config['api_key'] ?? '';
1544		- $host = $bot_pinecone_config['host'] ?? '';
1545		- $namespace = $bot_pinecone_config['namespace'] ?? '';
1546		- }
1547		- }
1548		-
1549		- if (empty($host) || empty($api_key)) {
1550		- return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
1551		- }
1552		-
1553		- $base_vector_id = md5($source_url);
1554		- $vectors_to_delete = array();
1555		-
1556		- // Add the original single-vector ID (for non-chunked content)
1557		- $vectors_to_delete[] = $base_vector_id;
1558		-
1559		- // Pinecone /vectors/list is a GET endpoint with query-string parameters; a POST here returns a
1560		- // non-200 silently and we end up only deleting the base vector, leaving chunks orphaned.
1561		- $query_params = array(
1562		- 'prefix' => $base_vector_id . '_chunk_',
1563		- 'limit' => 100,
1564		- );
1565		- if (!empty($namespace)) {
1566		- $query_params['namespace'] = $namespace;
1567		- }
1568		-
1569		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
1570		-
1571		- // Paginate in case a URL has more than 100 chunks.
1572		- do {
1573		- $list_response = wp_remote_get($list_url, array(
1574		- 'headers' => array(
1575		- 'Api-Key' => $api_key,
1576		- 'accept' => 'application/json',
1577		- ),
1578		- 'timeout' => 30,
1579		- ));
1580		-
1581		- if (is_wp_error($list_response) || wp_remote_retrieve_response_code($list_response) !== 200) {
1582		- break;
1583		- }
1584		-
1585		- $list_data = json_decode(wp_remote_retrieve_body($list_response), true);
1586		- if (!empty($list_data['vectors'])) {
1587		- foreach ($list_data['vectors'] as $vector) {
1588		- if (isset($vector['id'])) {
1589		- $vectors_to_delete[] = $vector['id'];
1590		- }
1591		- }
1592		- }
1593		-
1594		- $next_token = $list_data['pagination']['next'] ?? '';
1595		- if (empty($next_token)) {
1596		- break;
1597		- }
1598		-
1599		- $query_params['paginationToken'] = $next_token;
1600		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
1601		- } while (true);
1602		-
1603		- if (empty($vectors_to_delete)) {
1604		- //error_log('[MXCHAT-CHUNK-DELETE] No vectors found to delete');
1605		- return true;
1606		- }
1607		-
1608		- //error_log('[MXCHAT-CHUNK-DELETE] Deleting ' . count($vectors_to_delete) . ' vectors from Pinecone');
1609		-
1610		- // Delete vectors
1611		- $delete_url = "https://{$host}/vectors/delete";
1612		-
1613		- $delete_body = array(
1614		- 'ids' => $vectors_to_delete
1615		- );
1616		-
1617		- if (!empty($namespace)) {
1618		- $delete_body['namespace'] = $namespace;
1619		- }
1620		-
1621		- $delete_response = wp_remote_post($delete_url, array(
1622		- 'headers' => array(
1623		- 'Api-Key' => $api_key,
1624		- 'accept' => 'application/json',
1625		- 'content-type' => 'application/json'
1626		- ),
1627		- 'body' => wp_json_encode($delete_body),
1628		- 'timeout' => 30
1629		- ));
1630		-
1631		- if (is_wp_error($delete_response)) {
1632		- return $delete_response;
1633		- }
1634		-
1635		- $response_code = wp_remote_retrieve_response_code($delete_response);
1636		- if ($response_code !== 200) {
1637		- return new WP_Error('pinecone_delete', 'Failed to delete vectors: HTTP ' . $response_code);
1638		- }
1639		-
1640		- return true;
1641		-}
1642		-
1643		-/**
1644		- * Delete all chunks for a URL from WordPress database
1645		- */
1646		-private static function delete_wordpress_chunks_by_url($source_url) {
1647		- global $wpdb;
1648		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1649		-
1650		- // Delete all rows with this source_url (handles both chunked and non-chunked)
1651		- $result = $wpdb->delete(
1652		- $table_name,
1653		- array('source_url' => $source_url),
1654		- array('%s')
1655		- );
1656		-
1657		- if ($result === false) {
1658		- return new WP_Error('database_delete', 'Failed to delete chunks: ' . $wpdb->last_error);
1659		- }
1660		-
1661		- //error_log('[MXCHAT-CHUNK-DELETE] Deleted ' . $result . ' rows from WordPress DB');
1662		- return true;
1663		-}
1664		-
1665		-/**
1666		- * Hybrid keyword boost (plan-38ffa1): detect whether the WP-DB knowledge
1667		- * table can serve the keyword leg via a MySQL FULLTEXT index, creating the
1668		- * index if needed. Detection runs once and caches the answer in the
1669		- * mxchat_hybrid_keyword_capability option ('fulltext' | 'like'); pass
1670		- * $force to re-detect. LIKE is the graceful fallback for shared hosts
1671		- * whose ALTER fails — the feature works either way, FULLTEXT just ranks
1672		- * better and scales.
1673		- *
1674		- * @param bool $force Re-run detection even if a cached answer exists.
1675		- * @return string 'fulltext' or 'like'
1676		- */
1677		-public static function mxchat_hybrid_detect_capability($force = false) {
1678		- $cached = get_option('mxchat_hybrid_keyword_capability', '');
1679		- if (!$force && in_array($cached, array('fulltext', 'like'), true)) {
1680		- return $cached;
1681		- }
1682		-
1683		- global $wpdb;
1684		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
1685		-
1686		- $index_exists = $wpdb->get_var("SHOW INDEX FROM {$table} WHERE Key_name = 'mxchat_content_ft'");
1687		- if (!$index_exists) {
1688		- // Suppress the visible error on hosts where this is not permitted —
1689		- // failure is an expected, handled outcome (LIKE fallback).
1690		- $suppress = $wpdb->suppress_errors(true);
1691		- $wpdb->query("ALTER TABLE {$table} ADD FULLTEXT INDEX mxchat_content_ft (article_content)");
1692		- $wpdb->suppress_errors($suppress);
1693		- $index_exists = $wpdb->get_var("SHOW INDEX FROM {$table} WHERE Key_name = 'mxchat_content_ft'");
1694		- }
1695		-
1696		- $capability = $index_exists ? 'fulltext' : 'like';
1697		- update_option('mxchat_hybrid_keyword_capability', $capability);
1698		- return $capability;
1699		-}
1700		-
1701		-/**
1702		- * Public entry for re-embedding already-stored content in place (wp mxchat
1703		- * rtl-repair, plan d1e6f7). Thin wrapper so the repair CLI gets the exact
1704		- * provider routing the import path uses — the repaired vector must come from
1705		- * the same model family the bot indexes with, or retrieval stays broken.
1706		- */
1707		-public static function regenerate_embedding($text, $api_key, $bot_id = 'default') {
1708		- return self::generate_embedding($text, $api_key, $bot_id);
1709		-}
1710		-
1711		-/**
1712		- * Restore logical character order in PDF-extracted RTL text (plan 32bf9e).
1713		- *
1714		- * The bundled Smalot parser only un-reverses text runs tagged with the
1715		- * ReversedChars marked-content operator (Word emits it; LibreOffice and most
1716		- * other producers do not), so their Hebrew/Arabic PDFs extract in visual
1717		- * (reversed) order and embed/search as garbage. This is OUR post-processing
1718		- * seam over getText() — the parser itself is never patched (it gets replaced
1719		- * wholesale on library updates).
1720		- *
1721		- * Heuristic and deliberately conservative, per line:
1722		- * - lines without strong RTL codepoints are untouched (a fully-Latin line in
1723		- * an RTL document therefore stays as extracted — accepted limitation);
1724		- * - Arabic presentation forms are a definitive visual-order signal (they only
1725		- * appear in shaped output): de-shape to base letters and reverse;
1726		- * - otherwise flip only on positive evidence — Hebrew final-letter position
1727		- * (a sofit at word START only happens in reversed text) or sentence
1728		- * punctuation position (leading in visual order, trailing in logical);
1729		- * - ambiguous lines are left alone: a conservative miss beats corrupting a
1730		- * Word-produced extraction the parser already handled (the double-flip
1731		- * guard this plan's approval named mandatory).
1732		- *
1733		- * @param string $text One extracted page string, straight from getText().
1734		- * @param string $context Caller tag for the Debug Mode entry (site + page).
1735		- * @return string Text with RTL lines restored to logical order.
1736		- */
1737		-public static function normalize_pdf_rtl($text, $context = '') {
1738		- if (!is_string($text) || '' === $text) {
1739		- return $text;
1740		- }
1741		- // Escape hatch for sites whose PDFs already extract logically.
1742		- if (!apply_filters('mxchat_pdf_rtl_normalize', true, $text)) {
1743		- return $text;
1744		- }
1745		- // Fast bail: nothing RTL anywhere in the page.
1746		- if (!preg_match('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $text)) {
1747		- return $text;
1748		- }
1749		-
1750		- $parts = preg_split('/(\R)/u', $text, -1, PREG_SPLIT_DELIM_CAPTURE);
1751		- if (false === $parts) {
1752		- return $text;
1753		- }
1754		-
1755		- $flipped_lines = 0;
1756		- $deshaped_lines = 0;
1757		- foreach ($parts as $i => $part) {
1758		- if ('' === $part || preg_match('/^\R$/u', $part)) {
1759		- continue;
1760		- }
1761		- $was_flipped = false;
1762		- $was_deshaped = false;
1763		- $new = self::pdf_rtl_normalize_line($part, $was_flipped, $was_deshaped);
1764		- if ($new !== $part) {
1765		- $parts[$i] = $new;
1766		- }
1767		- if ($was_flipped) {
1768		- $flipped_lines++;
1769		- }
1770		- if ($was_deshaped) {
1771		- $deshaped_lines++;
1772		- }
1773		- }
1774		-
1775		- if (($flipped_lines || $deshaped_lines) && class_exists('MxChat_Admin')) {
1776		- MxChat_Admin::mxchat_log_debug('pdf_rtl_normalized', 'RTL PDF text restored to logical order', array(
1777		- 'context' => (string) $context,
1778		- 'lines_flipped' => $flipped_lines,
1779		- 'lines_deshaped' => $deshaped_lines,
1780		- 'decision' => 'visual-order extraction detected',
1781		- ));
1782		- }
1783		-
1784		- return implode('', $parts);
1785		-}
1786		-
1787		-/**
1788		- * Normalize one line. Sets $flipped/$deshaped for the caller's debug entry.
1789		- */
1790		-private static function pdf_rtl_normalize_line($line, &$flipped, &$deshaped) {
1791		- $flipped = false;
1792		- $deshaped = false;
1793		-
1794		- if (!preg_match('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $line)) {
1795		- return $line;
1796		- }
1797		-
1798		- $has_forms = (bool) preg_match('/[\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $line);
1799		- $work = $line;
1800		- if ($has_forms) {
1801		- $work = strtr($work, self::pdf_rtl_deshape_map());
1802		- $deshaped = ($work !== $line);
1803		- }
1804		-
1805		- $verdict = 'ambiguous';
1806		- if ($has_forms) {
1807		- // Shaped glyph codepoints only exist in visual-order output.
1808		- $verdict = 'visual';
1809		- } else {
1810		- // Strong-direction dominance gate first: an LTR-dominant line with an
1811		- // embedded RTL word is not flip material.
1812		- $rtl_count = preg_match_all('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}]/u', $work, $m_rtl);
1813		- $ltr_count = preg_match_all('/[A-Za-z]/u', $work, $m_ltr);
1814		- if ($rtl_count < 1 || $rtl_count <= $ltr_count) {
1815		- return $line;
1816		- }
1817		-
1818		- // Hebrew final letters (ך ם ן ף ץ) end words in logical text; one at
1819		- // a word START (Hebrew letter follows, none precedes) is reversal
1820		- // evidence. Positional, so it survives the line being reversed.
1821		- $sofit_initial = preg_match_all('/(?<![\x{05D0}-\x{05EA}])[\x{05DA}\x{05DD}\x{05DF}\x{05E3}\x{05E5}](?=[\x{05D0}-\x{05EA}])/u', $work, $m_i);
1822		- $sofit_terminal = preg_match_all('/(?<=[\x{05D0}-\x{05EA}])[\x{05DA}\x{05DD}\x{05DF}\x{05E3}\x{05E5}](?![\x{05D0}-\x{05EA}])/u', $work, $m_t);
1823		- if ($sofit_initial > $sofit_terminal) {
1824		- $verdict = 'visual';
1825		- } elseif ($sofit_terminal > $sofit_initial) {
1826		- $verdict = 'logical';
1827		- } else {
1828		- // Sentence punctuation lands at the visual LEFT edge of an RTL
1829		- // line, i.e. the START of a visual-order extraction.
1830		- $trimmed = trim($work);
1831		- $starts_punct = (bool) preg_match('/^[.?!:;,]/u', $trimmed);
1832		- $ends_punct = (bool) preg_match('/[.?!:;,]$/u', $trimmed);
1833		- if ($starts_punct && !$ends_punct) {
1834		- $verdict = 'visual';
1835		- } elseif ($ends_punct && !$starts_punct) {
1836		- $verdict = 'logical';
1837		- }
1838		- }
1839		- }
1840		-
1841		- if ('visual' !== $verdict) {
1842		- // Ambiguous or logical: hand back the original line UNLESS we
1843		- // de-shaped (de-shaping alone is always safe — same letters, same
1844		- // order, un-ligated).
1845		- return $deshaped ? $work : $line;
1846		- }
1847		-
1848		- $flipped = true;
1849		- return self::pdf_rtl_flip_line($work);
1850		-}
1851		-
1852		-/**
1853		- * Reverse a visual-order line into logical order: full character reversal,
1854		- * mirror paired punctuation, then re-reverse embedded LTR runs (Latin words
1855		- * and digit sequences, incl. Arabic-Indic digits) so they stay readable.
1856		- */
1857		-private static function pdf_rtl_flip_line($line) {
1858		- $chars = preg_split('//u', $line, -1, PREG_SPLIT_NO_EMPTY);
1859		- if (false === $chars) {
1860		- return $line;
1861		- }
1862		- $reversed = implode('', array_reverse($chars));
1863		- $reversed = strtr($reversed, array(
1864		- '(' => ')', ')' => '(',
1865		- '[' => ']', ']' => '[',
1866		- '{' => '}', '}' => '{',
1867		- '<' => '>', '>' => '<',
1868		- ));
1869		- $restored = preg_replace_callback(
1870		- '/[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9}](?:[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9} .,\'"%\-:\/]*[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9}])?/u',
1871		- function ($m) {
1872		- $run = preg_split('//u', $m[0], -1, PREG_SPLIT_NO_EMPTY);
1873		- return false === $run ? $m[0] : implode('', array_reverse($run));
1874		- },
1875		- $reversed
1876		- );
1877		- return null === $restored ? $reversed : $restored;
1878		-}
1879		-
1880		-/**
1881		- * Arabic presentation forms (A + B) -> base letters. Built once from range
1882		- * specs rather than ~120 hand-written literal entries; every codepoint in a
1883		- * range maps to the same base sequence (isolated/final/initial/medial forms
1884		- * of one letter are contiguous in the FE70 block).
1885		- */
1886		-private static function pdf_rtl_deshape_map() {
1887		- static $map = null;
1888		- if (null !== $map) {
1889		- return $map;
1890		- }
1891		- $ranges = array(
1892		- // Form B harakat (each pair = standalone + tatweel-joined form).
1893		- array(0xFE70, 0xFE71, array(0x064B)), array(0xFE72, 0xFE72, array(0x064C)),
1894		- array(0xFE74, 0xFE74, array(0x064D)), array(0xFE76, 0xFE77, array(0x064E)),
1895		- array(0xFE78, 0xFE79, array(0x064F)), array(0xFE7A, 0xFE7B, array(0x0650)),
1896		- array(0xFE7C, 0xFE7D, array(0x0651)), array(0xFE7E, 0xFE7F, array(0x0652)),
1897		- // Form B letters.
1898		- array(0xFE80, 0xFE80, array(0x0621)), array(0xFE81, 0xFE82, array(0x0622)),
1899		- array(0xFE83, 0xFE84, array(0x0623)), array(0xFE85, 0xFE86, array(0x0624)),
1900		- array(0xFE87, 0xFE88, array(0x0625)), array(0xFE89, 0xFE8C, array(0x0626)),
1901		- array(0xFE8D, 0xFE8E, array(0x0627)), array(0xFE8F, 0xFE92, array(0x0628)),
1902		- array(0xFE93, 0xFE94, array(0x0629)), array(0xFE95, 0xFE98, array(0x062A)),
1903		- array(0xFE99, 0xFE9C, array(0x062B)), array(0xFE9D, 0xFEA0, array(0x062C)),
1904		- array(0xFEA1, 0xFEA4, array(0x062D)), array(0xFEA5, 0xFEA8, array(0x062E)),
1905		- array(0xFEA9, 0xFEAA, array(0x062F)), array(0xFEAB, 0xFEAC, array(0x0630)),
1906		- array(0xFEAD, 0xFEAE, array(0x0631)), array(0xFEAF, 0xFEB0, array(0x0632)),
1907		- array(0xFEB1, 0xFEB4, array(0x0633)), array(0xFEB5, 0xFEB8, array(0x0634)),
1908		- array(0xFEB9, 0xFEBC, array(0x0635)), array(0xFEBD, 0xFEC0, array(0x0636)),
1909		- array(0xFEC1, 0xFEC4, array(0x0637)), array(0xFEC5, 0xFEC8, array(0x0638)),
1910		- array(0xFEC9, 0xFECC, array(0x0639)), array(0xFECD, 0xFED0, array(0x063A)),
1911		- array(0xFED1, 0xFED4, array(0x0641)), array(0xFED5, 0xFED8, array(0x0642)),
1912		- array(0xFED9, 0xFEDC, array(0x0643)), array(0xFEDD, 0xFEE0, array(0x0644)),
1913		- array(0xFEE1, 0xFEE4, array(0x0645)), array(0xFEE5, 0xFEE8, array(0x0646)),
1914		- array(0xFEE9, 0xFEEC, array(0x0647)), array(0xFEED, 0xFEEE, array(0x0648)),
1915		- array(0xFEEF, 0xFEF0, array(0x0649)), array(0xFEF1, 0xFEF4, array(0x064A)),
1916		- // Form B lam-alef ligatures decompose to two letters.
1917		- array(0xFEF5, 0xFEF6, array(0x0644, 0x0622)), array(0xFEF7, 0xFEF8, array(0x0644, 0x0623)),
1918		- array(0xFEF9, 0xFEFA, array(0x0644, 0x0625)), array(0xFEFB, 0xFEFC, array(0x0644, 0x0627)),
1919		- // Form A: Persian / Urdu letters in common use.
1920		- array(0xFB56, 0xFB59, array(0x067E)), array(0xFB66, 0xFB69, array(0x0679)),
1921		- array(0xFB7A, 0xFB7D, array(0x0686)), array(0xFB88, 0xFB89, array(0x0688)),
1922		- array(0xFB8A, 0xFB8B, array(0x0698)), array(0xFB8E, 0xFB91, array(0x06A9)),
1923		- array(0xFB92, 0xFB95, array(0x06AF)), array(0xFBA6, 0xFBA9, array(0x06C1)),
1924		- array(0xFBAA, 0xFBAD, array(0x06BE)), array(0xFBAE, 0xFBAF, array(0x06D2)),
1925		- array(0xFBFC, 0xFBFF, array(0x06CC)),
1926		- );
1927		- $map = array();
1928		- foreach ($ranges as $range) {
1929		- $base = '';
1930		- foreach ($range[2] as $cp) {
1931		- $base .= self::pdf_rtl_cp_to_utf8($cp);
1932		- }
1933		- for ($cp = $range[0]; $cp <= $range[1]; $cp++) {
1934		- $map[self::pdf_rtl_cp_to_utf8($cp)] = $base;
1935		- }
1936		- }
1937		- return $map;
1938		-}
1939		-
1940		-/**
1941		- * Codepoint to UTF-8 without ext-intl / mbstring entity tricks (PHP 7.2 floor).
1942		- */
1943		-private static function pdf_rtl_cp_to_utf8($cp) {
1944		- if ($cp < 0x80) {
1945		- return chr($cp);
1946		- }
1947		- if ($cp < 0x800) {
1948		- return chr(0xC0 | ($cp >> 6)) . chr(0x80 | ($cp & 0x3F));
1949		- }
1950		- if ($cp < 0x10000) {
1951		- return chr(0xE0 | ($cp >> 12)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
1952		- }
1953		- return chr(0xF0 | ($cp >> 18)) . chr(0x80 | (($cp >> 12) & 0x3F)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
1954		-}
1955		-}
	1	+<?php
	2	+if (!defined('ABSPATH')) {
	3	+ exit; // Exit if accessed directly
	4	+}
	5	+
	6	+class MxChat_Utils {
	7	+
	8	+ /**
	9	+ * Submit content and its embedding to the database.
	10	+ *
	11	+ * @param string $content The content to be embedded.
	12	+ * @param string $source_url The source URL of the content.
	13	+ * @param string $api_key The API key used for generating embeddings.
	14	+ */
	15	+ public static function submit_content_to_db($content, $source_url, $api_key) {
	16	+ global $wpdb;
	17	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	18	+
	19	+ // Sanitize the source URL
	20	+ $source_url = esc_url_raw($source_url);
	21	+
	22	+ // Generate the embedding using the API key
	23	+ $embedding_vector = self::generate_embedding($content, $api_key);
	24	+
	25	+ if (is_array($embedding_vector)) {
	26	+ $embedding_vector_serialized = serialize($embedding_vector);
	27	+
	28	+ // Insert or update the record in the database
	29	+ $wpdb->replace(
	30	+ $table_name,
	31	+ array(
	32	+ 'article_content' => $content,
	33	+ 'embedding_vector' => $embedding_vector_serialized,
	34	+ 'source_url' => $source_url,
	35	+ ),
	36	+ array(
	37	+ '%s',
	38	+ '%s',
	39	+ '%s',
	40	+ )
	41	+ );
	42	+ } else {
	43	+ //error_log('Embedding generation failed for content from ' . $source_url);
	44	+ }
	45	+ }
	46	+
	47	+ /**
	48	+ * Generate an embedding for the given text.
	49	+ *
	50	+ * @param string $text The text to be embedded.
	51	+ * @param string $api_key The API key used for generating embeddings.
	52	+ * @return array|null The embedding vector or null on failure.
	53	+ */
	54	+ private static function generate_embedding($text, $api_key) {
	55	+ $endpoint = 'https://api.openai.com/v1/embeddings';
	56	+
	57	+ $body = wp_json_encode([
	58	+ 'input' => $text,
	59	+ 'model' => 'text-embedding-ada-002'
	60	+ ]);
	61	+
	62	+ $args = [
	63	+ 'body' => $body,
	64	+ 'headers' => [
	65	+ 'Content-Type' => 'application/json',
	66	+ 'Authorization' => 'Bearer ' . $api_key,
	67	+ ],
	68	+ 'timeout' => 60,
	69	+ 'redirection' => 5,
	70	+ 'blocking' => true,
	71	+ 'httpversion' => '1.0',
	72	+ 'sslverify' => true,
	73	+ ];
	74	+
	75	+ $response = wp_remote_post($endpoint, $args);
	76	+
	77	+ if (is_wp_error($response)) {
	78	+ //error_log('Error generating embedding: ' . $response->get_error_message());
	79	+ return null;
	80	+ }
	81	+
	82	+ $response_body = json_decode(wp_remote_retrieve_body($response), true);
	83	+
	84	+ if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
	85	+ return $response_body['data'][0]['embedding'];
	86	+ } else {
	87	+ //error_log('Invalid response received from embedding API.');
	88	+ return null;
	89	+ }
	90	+ }
	91	+}
	92	+
	93	+?>