← All changes | includes/class-mxchat-utils.php +93 -2162 3.2.21 → 1.0.8 View file →

		@@ -1,2162 +1,93 @@
1		-<?php
2		-if (!defined('ABSPATH')) {
3		- exit; // Exit if accessed directly
4		-}
5		-
6		-class MxChat_Utils {
7		-
8		-/**
9		- * True while a storage routine is re-writing an entry it is about to re-add
10		- * (submit_chunked_content's clean-slate delete). delete_chunks_for_url() skips
11		- * the Vector Store mirror-delete while set — an internal re-store is not an
12		- * entry removal, and mirroring it would delete-then-reupload the entry's file
13		- * on every chunked save (plan 15b5c6).
14		- */
15		-private static $vectorstore_mirror_suspended = false;
16		-
17		-/**
18		- * Validate a client-supplied session id (plan-mxchat-20260731-d42bec).
19		- *
20		- * sanitize_text_field() — which every session_id read site used before this —
21		- * preserves '/' and '..'. Harmless where the value is only an option or
22		- * transient key suffix, but mxchat_send_delayed_transcript() interpolates it
23		- * into a filesystem path, so '../../../../path/x' wrote, emailed and deleted a
24		- * file outside the uploads dir.
25		- *
26		- * REJECTS rather than rewrites: a silently-stripped id would orphan the
27		- * conversation it belongs to, which is harder to diagnose than a clean refusal.
28		- * Returns '' for anything malformed, so call sites fall into the empty-session
29		- * error paths they already have.
30		- *
31		- * The generator only ever emits 'mxchat_chat_' + 32 hex chars
32		- * (class-mxchat-integrator.php, js/chat-script.js), so this is not restrictive
33		- * in practice. Length ceiling is deliberate — session ids are also used as
34		- * option-name suffixes, and WP option names cap at 191 chars.
35		- *
36		- * @param mixed $raw Raw request value.
37		- * @return string The id if well-formed, '' otherwise.
38		- */
39		-public static function sanitize_session_id($raw) {
40		- if (!is_scalar($raw)) {
41		- return '';
42		- }
43		- $val = trim((string) $raw);
44		- if ($val === '') {
45		- return '';
46		- }
47		- return preg_match('/\A[A-Za-z0-9_-]{1,128}\z/', $val) ? $val : '';
48		-}
49		-
50		-/**
51		- * Sanitize the lead-capture consent-checkbox label (plan b062c4).
52		- *
53		- * The label is owner-supplied and renders inside the widget's email form, so
54		- * this is a security boundary, not a formatting nicety. One explicit
55		- * allowlist, used at BOTH save time (options.php sanitize + autosave AJAX)
56		- * and render time (widget form, admin surfaces) so the two can never drift:
57		- * an anchor — the whole point is "I agree to the <a>Privacy Policy</a>" —
58		- * plus inline emphasis. No block tags, no images, no style attributes.
59		- *
60		- * The stored consent record keeps this exact sanitized string as "the text
61		- * the visitor saw", so it must be deterministic: same input, same output,
62		- * whichever path ran it.
63		- *
64		- * @param mixed $raw Owner-entered label.
65		- * @return string Sanitized label, capped at 1000 chars.
66		- */
67		-public static function sanitize_consent_label($raw) {
68		- if (!is_scalar($raw)) {
69		- return '';
70		- }
71		-
72		- $allowed = array(
73		- 'a' => array(
74		- 'href' => true,
75		- 'title' => true,
76		- 'target' => true,
77		- 'rel' => true,
78		- ),
79		- 'strong' => array(),
80		- 'em' => array(),
81		- 'br' => array(),
82		- );
83		-
84		- $label = wp_kses(trim((string) $raw), $allowed);
85		-
86		- return mb_substr($label, 0, 1000);
87		-}
88		-
89		-/**
90		- * Per-request cache for get_session_history(). Mirrors get_option()'s
91		- * request-scoped caching, which the mxchat_history_ option reads got for
92		- * free before plan 839c4c moved history reads onto the transcripts table.
93		- */
94		-private static $history_cache = array();
95		-
96		-/**
97		- * Session chat history read from the transcripts table, in the exact array
98		- * shape the legacy mxchat_history_<sid> option stored (plan 839c4c). The
99		- * option was a second copy of state the table already held — measured
100		- * byte-identical in role/content/order on 174 of 177 real sessions, with
101		- * the table a superset on the rest — at up to 64 KB per option row. The
102		- * table is now the single store; nothing writes the option any more.
103		- *
104		- * Shape notes, load-bearing for the consumers:
105		- * - id: the transcripts row id (int). Integer ids make the pollers'
106		- * ">" comparisons correct where the old uniqid() strings only worked by
107		- * accident of hex ordering.
108		- * - timestamp: milliseconds, derived from the table's second-resolution GMT
109		- * column (x1000). Consumers comparing against a real-millisecond client
110		- * cutoff MUST floor the cutoff to the second and err inclusive — see the
111		- * persistence-off filters in class-mxchat-integrator.php.
112		- * - agent_name: the row's user_identifier, which the writer sets to the
113		- * same displayed_name value the option carried (agent name when present,
114		- * else email, else identifier).
115		- *
116		- * Public and static so mxchat-woo / mxchat-forms can call the same accessor
117		- * as core, guarded with method_exists against an older mxchat-basic.
118		- *
119		- * @param string $session_id
120		- * @return array[] Chronological entries: id, role, content, timestamp, agent_name.
121		- */
122		-public static function get_session_history($session_id) {
123		- global $wpdb;
124		-
125		- $session_id = self::sanitize_session_id($session_id);
126		- if ($session_id === '') {
127		- return array();
128		- }
129		-
130		- if (array_key_exists($session_id, self::$history_cache)) {
131		- return self::$history_cache[$session_id];
132		- }
133		-
134		- $table = $wpdb->prefix . 'mxchat_chat_transcripts';
135		-
136		- // No SHOW TABLES guard: this is the chat hot path and the table is
137		- // created on activation (with an admin-load safety net). A genuinely
138		- // missing table fails the query and yields the same empty history the
139		- // old option read produced on a fresh session.
140		- $rows = $wpdb->get_results(
141		- $wpdb->prepare(
142		- "SELECT id, role, message, user_identifier, timestamp
143		- FROM `$table` WHERE session_id = %s ORDER BY id ASC",
144		- $session_id
145		- ),
146		- ARRAY_A
147		- );
148		-
149		- $history = array();
150		- if (is_array($rows)) {
151		- foreach ($rows as $row) {
152		- // The column stores GMT (current_time('mysql', 1) at the writer),
153		- // so pin the parse to UTC rather than the site timezone.
154		- $ts = strtotime($row['timestamp'] . ' +0000');
155		- $history[] = array(
156		- 'id' => (int) $row['id'],
157		- 'role' => (string) $row['role'],
158		- 'content' => (string) $row['message'],
159		- 'timestamp' => ($ts ? $ts : 0) * 1000,
160		- 'agent_name' => (string) $row['user_identifier'],
161		- );
162		- }
163		- }
164		-
165		- self::$history_cache[$session_id] = $history;
166		-
167		- return $history;
168		-}
169		-
170		-/**
171		- * Drop the cached history for one session (or all). The writer calls this
172		- * after every insert so a later read in the same request — e.g. the AI
173		- * context build that follows saving the user's message — sees the new row,
174		- * matching the read-your-own-write behavior update_option() gave the old
175		- * option copy.
176		- *
177		- * @param string|null $session_id Null flushes everything (test seam).
178		- */
179		-public static function flush_session_history_cache($session_id = null) {
180		- if ($session_id === null) {
181		- self::$history_cache = array();
182		- return;
183		- }
184		-
185		- unset(self::$history_cache[(string) $session_id]);
186		-}
187		-
188		-/**
189		- * Most recipients the Notification Email field will accept (plan 2f131a).
190		- * A settings field is not a mailing list.
191		- */
192		-const NOTIFICATION_EMAIL_MAX = 5;
193		-
194		-/**
195		- * Parse the Notification Email field into a list of recipients (plan 2f131a).
196		- *
197		- * THE TRAP THIS EXISTS TO CLOSE: sanitize_email() cannot be the validator for
198		- * this field, because its output for the failing input is VALID. WordPress
199		- * strips the separator and the surplus '@' and concatenates the remains:
200		- *
201		- * support@acme.com, sales@acme.com -> support@acme.comsalesacme.com
202		- *
203		- * and is_email() then returns true on that. So every guard in the plugin passed,
204		- * the address was stored, the autosave ticked green, and both the new-session
205		- * notification and the auto-emailed transcript went to a domain that does not
206		- * exist — with no error anywhere. Validating the RAW part BEFORE sanitizing is
207		- * the whole point; reversing those two lines silently restores the bug.
208		- *
209		- * All-or-nothing by design: if any entry is bad the caller must store NOTHING.
210		- * A partial accept — keeping the good addresses and dropping the bad one — is
211		- * the same defect in a new costume, because the owner still believes everyone
212		- * on their list is being notified.
213		- *
214		- * @param mixed $raw Raw field value, exactly as submitted.
215		- * @return array{emails: string[], error: string} Empty emails + empty error
216		- * means the field was empty.
217		- */
218		-public static function parse_notification_emails($raw) {
219		- $out = array('emails' => array(), 'error' => '');
220		-
221		- if (!is_scalar($raw)) {
222		- $out['error'] = __('The notification email could not be read.', 'mxchat');
223		- return $out;
224		- }
225		-
226		- $raw = trim((string) $raw);
227		- if ($raw === '') {
228		- return $out; // genuinely empty — the caller falls back to admin_email
229		- }
230		-
231		- $seen = array();
232		- foreach (preg_split('/[,;]/', $raw) as $part) {
233		- $part = trim($part);
234		- if ($part === '') {
235		- // A trailing or doubled separator carries no address, so skipping it
236		- // cannot silently drop a recipient. This is the ONLY thing tolerated.
237		- continue;
238		- }
239		-
240		- // RAW first. See the note above — order is load-bearing.
241		- $clean = is_email($part) ? sanitize_email($part) : '';
242		- if ($clean === '' || !is_email($clean)) {
243		- return array(
244		- 'emails' => array(),
245		- 'error' => sprintf(
246		- /* translators: %s: the email address the owner typed. */
247		- __('"%s" is not a valid email address, so nothing was saved. Separate multiple addresses with a comma.', 'mxchat'),
248		- esc_html($part)
249		- ),
250		- );
251		- }
252		-
253		- $key = strtolower($clean);
254		- if (isset($seen[$key])) {
255		- continue; // same address twice would simply mail them twice
256		- }
257		- $seen[$key] = true;
258		- $out['emails'][] = $clean;
259		- }
260		-
261		- if (count($out['emails']) > self::NOTIFICATION_EMAIL_MAX) {
262		- return array(
263		- 'emails' => array(),
264		- 'error' => sprintf(
265		- /* translators: %d: maximum number of notification recipients. */
266		- __('Enter at most %d email addresses, separated by commas.', 'mxchat'),
267		- self::NOTIFICATION_EMAIL_MAX
268		- ),
269		- );
270		- }
271		-
272		- return $out;
273		-}
274		-
275		-/**
276		- * The stored recipient list, ready to hand to wp_mail() (plan 2f131a).
277		- *
278		- * Fallback rule, and it is narrow on purpose: an EMPTY field falls back to the
279		- * site admin address, because that is the documented behaviour and an owner who
280		- * never filled the field in still wants their notifications. A field holding
281		- * something unusable does NOT fall back — it sends nowhere, exactly as before
282		- * this plan. Falling back on bad input would mean a typo silently redirects a
283		- * store's transcripts to a different mailbox than the one on screen.
284		- *
285		- * @param array|null $options mxchat_transcripts_options, or null to read it.
286		- * @return string[] Recipients; empty means do not send.
287		- */
288		-public static function notification_recipients($options = null) {
289		- if (!is_array($options)) {
290		- $options = get_option('mxchat_transcripts_options', array());
291		- if (!is_array($options)) {
292		- $options = array();
293		- }
294		- }
295		-
296		- $raw = isset($options['mxchat_notification_email']) ? $options['mxchat_notification_email'] : '';
297		- $raw = is_scalar($raw) ? trim((string) $raw) : '';
298		-
299		- if ($raw === '') {
300		- $admin = get_option('admin_email');
301		- return is_email($admin) ? array($admin) : array();
302		- }
303		-
304		- $parsed = self::parse_notification_emails($raw);
305		- return $parsed['error'] === '' ? $parsed['emails'] : array();
306		-}
307		-
308		-/**
309		- * Centralized embedding model registry. Single source of truth for dimensions
310		- * and provider, so model-switch protection logic doesn't drift across files.
311		- */
312		-public static function embedding_model_registry() {
313		- return array(
314		- 'text-embedding-ada-002' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'Ada 2'),
315		- 'text-embedding-3-small' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'TE3 Small'),
316		- 'text-embedding-3-large' => array('dims' => 3072, 'provider' => 'openai', 'label' => 'TE3 Large'),
317		- 'voyage-3-large' => array('dims' => 2048, 'provider' => 'voyage', 'label' => 'Voyage-3 Large'),
318		- 'gemini-embedding-001' => array('dims' => 1536, 'provider' => 'gemini', 'label' => 'Gemini Embedding'),
319		- );
320		-}
321		-
322		-public static function embedding_model_dimensions($model) {
323		- $registry = self::embedding_model_registry();
324		- return isset($registry[$model]) ? (int) $registry[$model]['dims'] : 0;
325		-}
326		-
327		-public static function embedding_model_label($model) {
328		- if (is_string($model) && strpos($model, 'custom:') === 0) {
329		- /* translators: %s: the embedding model name configured on the custom provider */
330		- return sprintf(__('%s (custom provider)', 'mxchat'), substr($model, 7));
331		- }
332		- $registry = self::embedding_model_registry();
333		- return isset($registry[$model]) ? $registry[$model]['label'] : $model;
334		-}
335		-
336		-/**
337		- * Returns the model that was last used to actually write embeddings into the
338		- * KB. Differs from the user-selected setting once a switch has happened but
339		- * no re-embed has occurred yet — that's the mismatch state we warn about.
340		- */
341		-public static function get_active_embedding_model() {
342		- return get_option('mxchat_active_embedding_model', '');
343		-}
344		-
345		-/**
346		- * Stamp the model that produced the most recent successful embedding. Called
347		- * from generate_embedding() right after the API responds with a valid vector.
348		- */
349		-public static function stamp_active_embedding_model($model) {
350		- if (!empty($model) && $model !== self::get_active_embedding_model()) {
351		- update_option('mxchat_active_embedding_model', $model, false);
352		- }
353		-}
354		-
355		-/**
356		- * The model name the custom-provider embedding path will send, mirroring the
357		- * fallback chain the request itself uses: dedicated custom embedding model,
358		- * else the custom chat model, else 'default'. Single source shared by
359		- * generate_embedding_custom() and the mismatch-warning "selected" side so the
360		- * two can never drift (plan ae02cb).
361		- */
362		-public static function resolve_custom_embedding_model($options) {
363		- if (isset($options['custom_provider_embedding_model']) && trim((string) $options['custom_provider_embedding_model']) !== '') {
364		- return trim((string) $options['custom_provider_embedding_model']);
365		- }
366		- if (isset($options['custom_provider_model']) && trim((string) $options['custom_provider_model']) !== '') {
367		- return trim((string) $options['custom_provider_model']);
368		- }
369		- return 'default';
370		-}
371		-
372		-/**
373		- * The EFFECTIVE selected embedding model — what the next embed will actually
374		- * use. With custom-provider embeddings on this is the custom identity in the
375		- * same 'custom:<model>' form stamp_active_embedding_model() records, not the
376		- * inert standard dropdown value. Mismatch-warning comparisons must read this,
377		- * never $options['embedding_model'] directly — the dropdown cannot be
378		- * deselected, so reading it raw flags every correctly-configured custom setup.
379		- */
380		-public static function get_selected_embedding_model($options = null) {
381		- if (!is_array($options)) {
382		- $options = get_option('mxchat_options', array());
383		- }
384		- if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
385		- return 'custom:' . self::resolve_custom_embedding_model($options);
386		- }
387		- return $options['embedding_model'] ?? '';
388		-}
389		-
390		-/**
391		- * Extract the 11-character YouTube video ID from a URL, or '' if the URL is
392		- * not a single-video YouTube link. Single source of truth for both the KB
393		- * ingestion side and the chat render side — do not duplicate this parsing.
394		- * Channel, playlist, and search URLs deliberately return '' (only a URL that
395		- * identifies one video can be embedded).
396		- */
397		-public static function parse_youtube_id($url) {
398		- if (!is_string($url) || $url === '' || !preg_match('#^https?://#i', $url)) {
399		- return '';
400		- }
401		- $host = strtolower((string) wp_parse_url($url, PHP_URL_HOST));
402		- $host = preg_replace('/^(www|m)\./', '', $host);
403		- $path = (string) wp_parse_url($url, PHP_URL_PATH);
404		- $id = '';
405		- if ($host === 'youtu.be') {
406		- $segments = explode('/', ltrim($path, '/'));
407		- $id = $segments[0] ?? '';
408		- } elseif (in_array($host, array('youtube.com', 'youtube-nocookie.com'), true)) {
409		- if (preg_match('#^/(?:shorts|embed|live|v)/([A-Za-z0-9_-]+)#', $path, $m)) {
410		- $id = $m[1];
411		- } elseif ($path === '/watch') {
412		- parse_str((string) wp_parse_url($url, PHP_URL_QUERY), $query_vars);
413		- $id = isset($query_vars['v']) ? (string) $query_vars['v'] : '';
414		- }
415		- }
416		- $id = preg_replace('/[^A-Za-z0-9_-]/', '', (string) $id);
417		- return (strlen($id) === 11) ? $id : '';
418		-}
419		-
420		-/**
421		- * plan-mxchat-20260813-f52492 — video-card gating.
422		- *
423		- * Two standalone options (NOT mxchat_options — they skip the sanitize/autosave
424		- * traps entirely), read here so the gate in the integrator and the fields on
425		- * Knowledge -> Chunking & Retrieval can never disagree about a default.
426		- *
427		- * Master switch. Default ON: the card is existing behavior, and this is an
428		- * opt-out for owners who never want one, not a new feature to opt into.
429		- */
430		-public static function video_embed_enabled() {
431		- return get_option('mxchat_video_embed_enabled', 'on') === 'on';
432		-}
433		-
434		-/**
435		- * The video card's OWN confidence floor, as a 0-1 cosine — deliberately not
436		- * the site-wide Similarity Threshold (default 35). "Good enough to quote in
437		- * the answer" and "good enough to put a video on screen" are different
438		- * questions: retrieval is allowed to be generous because the model still
439		- * decides what to say, whereas the card is asserted to the visitor with no
440		- * such filter. Stored as an int percentage to match the site-wide slider's
441		- * convention; the default (55) sits above it on purpose.
442		- *
443		- * MXCHAT_VIDEO_EMBED_THRESHOLD_DEFAULT is the single source of that number.
444		- */
445		-public static function video_embed_threshold() {
446		- $stored = get_option('mxchat_video_embed_threshold', null);
447		- $percent = ($stored === null || $stored === '')
448		- ? MXCHAT_VIDEO_EMBED_THRESHOLD_DEFAULT
449		- : (int) $stored;
450		- if ($percent < 0) { $percent = 0; }
451		- if ($percent > 100) { $percent = 100; }
452		- // Cast: PHP evaluates 100/100 to int(1), so an unclamped return type would
453		- // vary with the stored value. Callers compare against a cosine — keep it float.
454		- return (float) $percent / 100;
455		-}
456		-
457		-/**
458		- * UPDATED: Submit or update content (and its embedding) in the database.
459		- * Stores in Pinecone if enabled, otherwise stores in WordPress DB.
460		- *
461		- * @param string $content The content to be embedded.
462		- * @param string $source_url The source URL of the content.
463		- * @param string $api_key The API key used for generating embeddings.
464		- * @param string $vector_id Optional vector ID for Pinecone (if not provided, will use md5 of URL)
465		- * @param string $bot_id The bot ID for multi-bot support
466		- * @param string $content_type The type of content (post, page, pdf, url, manual, product, etc.)
467		- * @return bool|WP_Error True on success, WP_Error on failure
468		- */
469		-public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
470		- global $wpdb;
471		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
472		-
473		- //error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ', Type: ' . $content_type . ')');
474		- //error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
475		-
476		- // Sanitize the source URL. Internal identities (mxchat:// manual docs,
477		- // upload:// file uploads) are NOT web URLs — esc_url_raw EMPTIES them
478		- // because its protocol list is statically cached and effectively
479		- // unfilterable (measured, plan 945406 / 0485e5) — sanitize those as text
480		- // so upserts stay keyed to a stable identity across re-imports.
481		- if (preg_match('#^(mxchat|upload)://#i', $source_url)) {
482		- $source_url = sanitize_text_field($source_url);
483		- } else {
484		- $source_url = esc_url_raw($source_url);
485		- }
486		-
487		- // Sanitize content_type
488		- $content_type = sanitize_key($content_type);
489		- if (empty($content_type)) {
490		- $content_type = 'content'; // Fallback for backwards compatibility
491		- }
492		-
493		- // Just ensure UTF-8 validity without aggressive escaping
494		- $safe_content = wp_check_invalid_utf8($content);
495		- // Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D)
496		- $safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
497		-
498		- // Check if chunking should be applied
499		- $chunker = MxChat_Chunker::from_settings();
500		- if ($chunker->should_chunk($safe_content)) {
501		- //error_log('[MXCHAT-DB] Content exceeds chunk threshold, using chunked submission');
502		- $chunk_result = self::submit_chunked_content($safe_content, $source_url, $api_key, $bot_id, $content_type, $chunker);
503		- // Vector Store mirror gets the entry WHOLE — one file per KB entry,
504		- // OpenAI chunks server-side; local chunking is our own embedding
505		- // concern and never reaches the store (plan 15b5c6).
506		- if ($chunk_result === true && class_exists('MxChat_Vectorstore_Manager')) {
507		- MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type);
508		- }
509		- return $chunk_result;
510		- }
511		-
512		- // UPDATED: Generate the embedding using bot-specific configuration
513		- $embedding_vector = self::generate_embedding($content, $api_key, $bot_id);
514		-
515		- if (!is_array($embedding_vector)) {
516		- // Surface the provider's real reason instead of a fixed string (4a7c0a).
517		- $reason = is_wp_error($embedding_vector)
518		- ? $embedding_vector->get_error_message()
519		- : 'Failed to generate embedding for content';
520		- return new WP_Error('embedding_failed', $reason);
521		- }
522		-
523		- //error_log('[MXCHAT-DB] Embedding generated successfully');
524		-
525		- // UPDATED: Check if Pinecone is enabled for this specific bot
526		- if (self::is_pinecone_enabled_for_bot($bot_id)) {
527		- //error_log('[MXCHAT-DB] Pinecone is enabled for bot ' . $bot_id . ' - using Pinecone storage');
528		- // Store in Pinecone only
529		- $result = self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id, $content_type);
530		- // If this URL previously stored as CHUNKED content and the new content fits in a
531		- // single vector, the upsert above only overwrote the base id — the old
532		- // md5(url)_chunk_N vectors would keep serving the stale text. Sweep them.
533		- // Only for a real source_url: chunk ids derive from it, and md5('') is shared
534		- // by legacy URL-less entries so a blind sweep there could hit other entries.
535		- if ($result === true && !empty($source_url)) {
536		- self::cleanup_pinecone_chunk_stragglers($source_url, $bot_id);
537		- }
538		- if ($result === true && class_exists('MxChat_Vectorstore_Manager')) {
539		- MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type);
540		- }
541		- return $result;
542		- } else {
543		- //error_log('[MXCHAT-DB] Pinecone not enabled for bot ' . $bot_id . ' - using WordPress storage');
544		- // Store in WordPress database only
545		- $embedding_vector_serialized = maybe_serialize($embedding_vector);
546		- $result = self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type);
547		- if ($result === true && class_exists('MxChat_Vectorstore_Manager')) {
548		- MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type);
549		- }
550		- return $result;
551		- }
552		-}
553		-
554		-/**
555		- * UPDATED: Check if Pinecone is enabled and properly configured for a specific bot
556		- */
557		-private static function is_pinecone_enabled_for_bot($bot_id = 'default') {
558		- // For default bot or when multi-bot is not active, use original method
559		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
560		- return self::is_pinecone_enabled();
561		- }
562		-
563		- // Get bot-specific Pinecone configuration
564		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
565		-
566		- if (empty($bot_pinecone_config)) {
567		- // Fallback to default configuration
568		- return self::is_pinecone_enabled();
569		- }
570		-
571		- $enabled_check = !empty($bot_pinecone_config['use_pinecone']) && $bot_pinecone_config['use_pinecone'];
572		- $api_key_check = !empty($bot_pinecone_config['api_key']);
573		- $host_check = !empty($bot_pinecone_config['host']);
574		-
575		- return $enabled_check && $api_key_check && $host_check;
576		-}
577		-
578		-/**
579		- * Check if Pinecone is enabled and properly configured (original method for default bot)
580		- */
581		-private static function is_pinecone_enabled() {
582		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
583		-
584		- if (empty($pinecone_options)) {
585		- return false;
586		- }
587		-
588		- $enabled_check = !empty($pinecone_options['mxchat_use_pinecone']) && $pinecone_options['mxchat_use_pinecone'] !== '0';
589		- $api_key_check = !empty($pinecone_options['mxchat_pinecone_api_key']);
590		- $host_check = !empty($pinecone_options['mxchat_pinecone_host']);
591		-
592		- return $enabled_check && $api_key_check && $host_check;
593		-}
594		-
595		-/**
596		- * UPDATED: Store content in Pinecone only with bot support
597		- */
598		-private static function store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
599		- //error_log('[MXCHAT-PINECONE] ===== Using Pinecone-only storage for bot ' . $bot_id . ' =====');
600		-
601		- // Get bot-specific Pinecone configuration
602		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
603		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
604		- $api_key = $pinecone_options['mxchat_pinecone_api_key'];
605		- $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
606		- $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
607		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
608		- } else {
609		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
610		- if (empty($bot_pinecone_config)) {
611		- // Fallback to default configuration
612		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
613		- $api_key = $pinecone_options['mxchat_pinecone_api_key'];
614		- $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
615		- $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
616		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
617		- } else {
618		- $api_key = $bot_pinecone_config['api_key'];
619		- $environment = ''; // Not used in new Pinecone API
620		- $index_name = ''; // Not used in new Pinecone API
621		- $namespace = $bot_pinecone_config['namespace'] ?? '';
622		- }
623		- }
624		-
625		- $result = self::store_in_pinecone_main(
626		- $embedding_vector,
627		- $content,
628		- $source_url,
629		- $api_key,
630		- $environment,
631		- $index_name,
632		- $vector_id,
633		- $bot_id,
634		- $namespace,
635		- $content_type
636		- );
637		-
638		- if (is_wp_error($result)) {
639		- //error_log('[MXCHAT-PINECONE] Pinecone storage failed for bot ' . $bot_id . ': ' . $result->get_error_message());
640		- return $result;
641		- }
642		-
643		- //error_log('[MXCHAT-PINECONE] Pinecone storage completed successfully for bot ' . $bot_id);
644		- return true;
645		-}
646		-
647		-/**
648		- * Store content in WordPress database with progressive fallback
649		- * UPDATED 2.5.6: Now includes content_type parameter
650		- */
651		-private static function store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type = 'content') {
652		- global $wpdb;
653		-
654		- //error_log('[MXCHAT-DB] ===== Using WordPress-only storage =====');
655		-
656		- // Sanitize content_type
657		- $content_type = sanitize_key($content_type);
658		- if (empty($content_type)) {
659		- $content_type = 'content'; // Fallback for backwards compatibility
660		- }
661		-
662		- // ===== FIXED: Generate unique identifier for manual content =====
663		- $original_source_url = $source_url;
664		- // Check if this is truly manual content (no URL at all) vs a real URL that filter_var rejects
665		- // filter_var(FILTER_VALIDATE_URL) rejects valid URLs with encoded chars, non-ASCII, fragments, etc.
666		- // Use a looser check: if it starts with http(s):// or has a scheme, it's a URL
667		- $has_url_scheme = !empty($source_url) && preg_match('#^https?://#i', $source_url);
668		- // upload:// identities (admin document/PDF uploads, plan 0485e5) are stable
669		- // and deduplicable — treat them like URLs so a re-upload UPDATES the row
670		- // instead of minting a fresh manual identity (which would duplicate).
671		- $has_stable_identity = $has_url_scheme || (!empty($source_url) && preg_match('#^upload://#i', $source_url));
672		- // Treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
673		- $is_legacy_mxchat_url = $has_url_scheme && strpos($source_url, 'mxchat.ai') !== false;
674		- $is_manual_content = empty($source_url) || $source_url === '' || !$has_stable_identity || $is_legacy_mxchat_url;
675		-
676		- if ($is_manual_content) {
677		- // Generate unique identifier for manual content to prevent overwrites
678		- $source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
679		- //error_log('[MXCHAT-DB] Generated unique ID for manual content: ' . $source_url);
680		- }
681		-
682		- // Only check for duplicates if we have a valid source URL (not manual content)
683		- $existing_id = null;
684		- if (!$is_manual_content) {
685		- $existing_id = $wpdb->get_var(
686		- $wpdb->prepare(
687		- "SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
688		- $source_url
689		- )
690		- );
691		- //error_log('[MXCHAT-DB] Checked for existing URL, found ID: ' . ($existing_id ?: 'none'));
692		- } else {
693		- //error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)');
694		- }
695		- // ===== END FIX =====
696		-
697		- // Progressive fallback mechanism for problematic content
698		- $attempt = 1;
699		- $max_attempts = 3;
700		- $current_content = $safe_content;
701		- $result = false;
702		-
703		- while ($attempt <= $max_attempts && $result === false) {
704		- try {
705		- if ($existing_id) {
706		- //error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
707		-
708		- // Update the existing row - UPDATED 2.5.6: Added content_type
709		- $result = $wpdb->update(
710		- $table_name,
711		- array(
712		- 'url' => $source_url,
713		- 'article_content' => $current_content,
714		- 'embedding_vector' => $embedding_vector_serialized,
715		- 'source_url' => $source_url,
716		- 'content_type' => $content_type,
717		- 'timestamp' => current_time('mysql'),
718		- ),
719		- array('id' => $existing_id),
720		- array('%s','%s','%s','%s','%s','%s'),
721		- array('%d')
722		- );
723		- } else {
724		- //error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
725		- //error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
726		-
727		- // Insert a new row - UPDATED 2.5.6: Added content_type
728		- $result = $wpdb->insert(
729		- $table_name,
730		- array(
731		- 'url' => $source_url, // Now unique for manual content
732		- 'article_content' => $current_content,
733		- 'embedding_vector' => $embedding_vector_serialized,
734		- 'source_url' => $source_url, // Now unique for manual content
735		- 'content_type' => $content_type,
736		- 'timestamp' => current_time('mysql'),
737		- ),
738		- array('%s','%s','%s','%s','%s','%s')
739		- );
740		- }
741		-
742		- if ($result === false) {
743		- //error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . ')');
744		- //error_log('[MXCHAT-DB] MySQL Error: ' . $wpdb->last_error);
745		- //error_log('[MXCHAT-DB] MySQL Error Number: ' . $wpdb->last_errno);
746		- //error_log('[MXCHAT-DB] Last Query: ' . substr($wpdb->last_query, 0, 500));
747		- //error_log('[MXCHAT-DB] Content length: ' . strlen($current_content) . ' bytes');
748		- //error_log('[MXCHAT-DB] Embedding vector length: ' . strlen($embedding_vector_serialized) . ' bytes');
749		-
750		- // Progressively apply more aggressive sanitization on failure
751		- if ($attempt === 1) {
752		- // First fallback: Use a more aggressive character filter and shorten
753		- $current_content = preg_replace('/[^\p{L}\p{N}\s.,;:!?()-]/u', '', $current_content);
754		- $current_content = substr($current_content, 0, 50000);
755		- } else if ($attempt === 2) {
756		- // Second fallback: Keep only alphanumeric and basic punctuation, shorten further
757		- $current_content = preg_replace('/[^a-zA-Z0-9\s.,;:!?()-]/u', '', $current_content);
758		- $current_content = substr($current_content, 0, 30000);
759		- }
760		-
761		- $attempt++;
762		- }
763		- } catch (Exception $e) {
764		- //error_log('[MXCHAT-DB] Exception during database operation: ' . $e->getMessage());
765		- $attempt++;
766		- }
767		- }
768		-
769		-if ($result === false) {
770		- //error_log('[MXCHAT-DB] All database operation attempts failed');
771		- //error_log('[MXCHAT-DB] Final MySQL Error: ' . $wpdb->last_error);
772		-
773		- $detailed_error = sprintf(
774		- 'Failed to store content in WordPress database after %d attempts. MySQL Error: %s (Error #%d). Content size: %d bytes, Embedding size: %d bytes',
775		- $max_attempts,
776		- $wpdb->last_error,
777		- $wpdb->last_errno,
778		- strlen($current_content),
779		- strlen($embedding_vector_serialized)
780		- );
781		-
782		- return new WP_Error('database_failed', $detailed_error);
783		-}
784		-
785		- //error_log('[MXCHAT-DB] WordPress database operation completed successfully (Attempt ' . ($attempt - 1) . ')');
786		- return true;
787		-}
788		-
789		-/**
790		- * UPDATED: Store content in Pinecone database with bot support
791		- * UPDATED 2.5.6: Now accepts content_type parameter
792		- */
793		-private static function store_in_pinecone_main($embedding_vector, $content, $url, $api_key, $environment, $index_name, $vector_id = null, $bot_id = 'default', $namespace = '', $content_type = 'content') {
794		- //error_log('[MXCHAT-PINECONE-MAIN] ===== Starting Pinecone storage for bot ' . $bot_id . ' =====');
795		-
796		- // ===== UPDATED: Handle manual content with unique vector IDs =====
797		- if ($vector_id) {
798		- // Use provided vector ID
799		- //error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
800		- } elseif (!empty($url) && preg_match('#^(https?|upload)://#i', $url)) {
801		- // For URLs — and stable upload:// identities (plan 0485e5) — use an
802		- // identity-derived ID so a re-import upserts the same vector.
803		- $vector_id = md5($url);
804		- //error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
805		- } else {
806		- // For manual content (empty/no URL scheme), generate unique ID
807		- $vector_id = 'manual_' . time() . '_' . substr(md5($content . microtime(true)), 0, 8);
808		- //error_log('[MXCHAT-PINECONE-MAIN] Generated unique vector ID for manual content: ' . $vector_id);
809		- }
810		- // ===== END UPDATE =====
811		-
812		- // Get host from bot-specific config or fallback to default
813		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
814		- $options = get_option('mxchat_pinecone_addon_options');
815		- $host = $options['mxchat_pinecone_host'] ?? '';
816		- } else {
817		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
818		- if (!empty($bot_pinecone_config)) {
819		- $host = $bot_pinecone_config['host'] ?? '';
820		- } else {
821		- $options = get_option('mxchat_pinecone_addon_options');
822		- $host = $options['mxchat_pinecone_host'] ?? '';
823		- }
824		- }
825		-
826		- //error_log('[MXCHAT-PINECONE-MAIN] Host: ' . $host);
827		- //error_log('[MXCHAT-PINECONE-MAIN] API key length: ' . strlen($api_key));
828		- //error_log('[MXCHAT-PINECONE-MAIN] Bot ID: ' . $bot_id);
829		- //error_log('[MXCHAT-PINECONE-MAIN] Namespace: ' . $namespace);
830		-
831		- if (empty($host)) {
832		- //error_log('[MXCHAT-PINECONE-MAIN] ERROR: Host is empty');
833		- return new WP_Error('pinecone_config', 'Pinecone host is not configured. Please set the host in your bot settings.');
834		- }
835		-
836		- // ===== UPDATED 2.5.6: Use passed content_type or determine from URL if not provided =====
837		- // Sanitize content_type
838		- $content_type = sanitize_key($content_type);
839		- if (empty($content_type)) {
840		- // Fallback to old detection logic for backwards compatibility
841		- $is_product = false;
842		- $content_type = 'manual'; // Default for manual content
843		-
844		- if (!empty($url) && preg_match('#^https?://#i', $url)) {
845		- $is_product = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
846		- $content_type = $is_product ? 'product' : 'content';
847		- }
848		- }
849		-
850		- //error_log('[MXCHAT-PINECONE-MAIN] Content type: ' . $content_type);
851		- // ===== END UPDATE =====
852		-
853		- $api_endpoint = "https://{$host}/vectors/upsert";
854		- //error_log('[MXCHAT-PINECONE-MAIN] API endpoint: ' . $api_endpoint);
855		-
856		- // UPDATED 2.5.6: Use provided content_type in metadata
857		- $metadata = array(
858		- 'text' => $content,
859		- 'source_url' => $url, // Can be empty for manual content
860		- 'type' => $content_type, // Now supports: post, page, pdf, url, manual, product, etc.
861		- 'last_updated' => time(),
862		- 'created_at' => time(), // Add creation timestamp
863		- 'bot_id' => $bot_id, // Add bot identification
864		- );
865		-
866		- $vector_data = array(
867		- 'id' => $vector_id,
868		- 'values' => $embedding_vector,
869		- 'metadata' => $metadata
870		- );
871		-
872		- $request_body = array(
873		- 'vectors' => array($vector_data)
874		- );
875		-
876		- // Add namespace if specified for multi-bot separation
877		- if (!empty($namespace)) {
878		- $request_body['namespace'] = $namespace;
879		- //error_log('[MXCHAT-PINECONE-MAIN] Using namespace: ' . $namespace);
880		- }
881		-
882		- //error_log('[MXCHAT-PINECONE-MAIN] Request body prepared (embedding dimensions: ' . count($embedding_vector) . ')');
883		-
884		- $response = wp_remote_post($api_endpoint, array(
885		- 'headers' => array(
886		- 'Api-Key' => $api_key,
887		- 'accept' => 'application/json',
888		- 'content-type' => 'application/json'
889		- ),
890		- 'body' => wp_json_encode($request_body),
891		- 'timeout' => 30,
892		- 'data_format' => 'body'
893		- ));
894		-
895		- if (is_wp_error($response)) {
896		- //error_log('[MXCHAT-PINECONE-MAIN] WordPress request error: ' . $response->get_error_message());
897		- return new WP_Error('pinecone_request', $response->get_error_message());
898		- }
899		-
900		- $response_code = wp_remote_retrieve_response_code($response);
901		- //error_log('[MXCHAT-PINECONE-MAIN] Response code: ' . $response_code);
902		-
903		- if ($response_code !== 200) {
904		- $body = wp_remote_retrieve_body($response);
905		- //error_log('[MXCHAT-PINECONE-MAIN] API error - Response body: ' . $body);
906		- return new WP_Error('pinecone_api', sprintf(
907		- 'Pinecone API error (HTTP %d): %s',
908		- $response_code,
909		- $body
910		- ));
911		- }
912		-
913		- $response_body = wp_remote_retrieve_body($response);
914		- //error_log('[MXCHAT-PINECONE-MAIN] Success response: ' . $response_body);
915		- //error_log('[MXCHAT-PINECONE-MAIN] Successfully stored in Pinecone for bot ' . $bot_id);
916		- //error_log('[MXCHAT-PINECONE-MAIN] ===== Pinecone storage complete =====');
917		-
918		- return true;
919		-}
920		-
921		-/**
922		- * Caller-side pre-flight for KB ingestion: can an embedding request be made
923		- * with these options, and which API key should travel downstream?
924		- *
925		- * Custom-provider-aware — generate_embedding() below routes to the custom
926		- * endpoint FIRST and ignores the passed cloud key entirely when
927		- * custom_provider_for_embeddings is on, so on that branch the only real
928		- * requirement is a Base URL. Ingestion callers that gated on a cloud API key
929		- * were killing keyless custom-embeddings sites (local Ollama / LM Studio
930		- * class) before the embed layer could route (plan cbd5fd).
931		- *
932		- * NOTE: reads $options['embedding_model'] raw on purpose — this mirrors
933		- * generate_embedding()'s own routing read, NOT the mismatch-banner's
934		- * "selected" chain (get_selected_embedding_model). The helper must predict
935		- * what the very next embed call will do, byte-for-byte.
936		- *
937		- * Decision only — callers keep their own error-surfacing shape (admin-notice
938		- * transient + redirect, wp_send_json_error, WP_Error, silent return).
939		- *
940		- * @param array|null $options Resolved options (bot-specific where the caller
941		- * has them); null loads the default bot's options.
942		- * @return array {
943		- * @type bool $ok Whether ingestion can proceed.
944		- * @type string $api_key Key to pass downstream ('' on the custom branch —
945		- * generate_embedding() ignores it there).
946		- * @type string $reason Human-readable blocker; '' when $ok.
947		- * @type string $provider Short provider label ('OpenAI', 'Voyage AI',
948		- * 'Google Gemini', 'Custom Provider').
949		- * }
950		- */
951		-public static function embedding_preflight($options = null) {
952		- if (!is_array($options)) {
953		- $options = get_option('mxchat_options');
954		- $options = is_array($options) ? $options : array();
955		- }
956		-
957		- // Custom branch mirrors generate_embedding()'s routing order (custom first).
958		- if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
959		- $base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
960		- if ($base_url === '') {
961		- return array(
962		- 'ok' => false,
963		- 'api_key' => '',
964		- // Same string generate_embedding_custom() returns for this state.
965		- 'reason' => __('Custom provider Base URL is not configured.', 'mxchat'),
966		- 'provider' => 'Custom Provider',
967		- );
968		- }
969		- return array('ok' => true, 'api_key' => '', 'reason' => '', 'provider' => 'Custom Provider');
970		- }
971		-
972		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
973		- if (strpos($selected_model, 'voyage') === 0) {
974		- $api_key = $options['voyage_api_key'] ?? '';
975		- $provider = 'Voyage AI';
976		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
977		- $api_key = $options['gemini_api_key'] ?? '';
978		- $provider = 'Google Gemini';
979		- } else {
980		- $api_key = $options['api_key'] ?? '';
981		- $provider = 'OpenAI';
982		- }
983		-
984		- if (empty($api_key)) {
985		- return array(
986		- 'ok' => false,
987		- 'api_key' => '',
988		- 'reason' => sprintf(
989		- /* translators: %s: embedding provider name */
990		- __('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
991		- $provider
992		- ),
993		- 'provider' => $provider,
994		- );
995		- }
996		-
997		- return array('ok' => true, 'api_key' => $api_key, 'reason' => '', 'provider' => $provider);
998		-}
999		-
1000		-/**
1001		- * Public QUERY-side entry point (plan 876edb). The chat pipeline's
1002		- * MxChat_Integrator::mxchat_generate_embedding() adapter routes through here
1003		- * so the query and index sides share ONE provider-routing implementation —
1004		- * the same endpoints, request bodies, and stamping semantics. The Integrator
1005		- * keeps its own error vocabulary by translating the WP_Error this returns
1006		- * (see the structured error data on every failure path below).
1007		- *
1008		- * @param string $text The text to be embedded.
1009		- * @param string $api_key Caller-resolved API key (per-bot on the query side).
1010		- * @param string $bot_id The bot ID for multi-bot support.
1011		- * @return array|WP_Error The embedding vector, or WP_Error carrying the reason.
1012		- */
1013		-public static function generate_query_embedding($text, $api_key, $bot_id = 'default') {
1014		- return self::generate_embedding($text, $api_key, $bot_id);
1015		-}
1016		-
1017		-/**
1018		- * UPDATED: Generate an embedding for the given text using bot-specific configuration.
1019		- *
1020		- * @param string $text The text to be embedded.
1021		- * @param string $api_key The API key used for generating embeddings.
1022		- * @param string $bot_id The bot ID for multi-bot support
1023		- * @return array|null The embedding vector or null on failure.
1024		- */
1025		-private static function generate_embedding($text, $api_key, $bot_id = 'default') {
1026		- // Get bot-specific options
1027		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
1028		- $options = get_option('mxchat_options');
1029		- } else {
1030		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
1031		- $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
1032		- }
1033		-
1034		- // Opt-in: when the custom provider is selected for embeddings, route the KB
1035		- // INDEX side through the same custom endpoint the query side uses, so stored
1036		- // vectors and query vectors come from the same model. Default-off behavior
1037		- // below is untouched.
1038		- if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
1039		- $custom = self::generate_embedding_custom($text, $options);
1040		- // The custom path already returns a human-readable error string —
1041		- // carry it instead of collapsing to null (plan 4a7c0a). The 'custom'
1042		- // branch marker lets the Integrator adapter map the string back onto
1043		- // its own error codes (876edb).
1044		- return is_array($custom) ? $custom : new WP_Error('embedding_failed', (string) $custom, array('branch' => 'custom'));
1045		- }
1046		-
1047		- $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
1048		-
1049		- // Determine endpoint and API key based on model
1050		- if (strpos($selected_model, 'voyage') === 0) {
1051		- $endpoint = 'https://api.voyageai.com/v1/embeddings';
1052		- $api_key = $options['voyage_api_key'] ?? '';
1053		- } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
1054		- $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
1055		- $api_key = $options['gemini_api_key'] ?? '';
1056		- } else {
1057		- $endpoint = 'https://api.openai.com/v1/embeddings';
1058		- // Prefer the caller-resolved key when one was passed — the query side
1059		- // resolves per-bot keys at its call sites (integrator adapter, 876edb).
1060		- // Index callers pass the preflight key, which equals this options read,
1061		- // so nothing changes for them.
1062		- $api_key = !empty($api_key) ? $api_key : ($options['api_key'] ?? '');
1063		- }
1064		-
1065		- // Prepare request body based on provider
1066		- if (strpos($selected_model, 'gemini-embedding') === 0) {
1067		- // Gemini API format
1068		- $request_body = [
1069		- 'model' => 'models/' . $selected_model,
1070		- 'content' => [
1071		- 'parts' => [
1072		- ['text' => $text]
1073		- ]
1074		- ],
1075		- 'outputDimensionality' => 1536
1076		- ];
1077		-
1078		- // Prepare headers for Gemini (API key as query parameter)
1079		- $endpoint .= '?key=' . $api_key;
1080		- $headers = [
1081		- 'Content-Type' => 'application/json'
1082		- ];
1083		- } else {
1084		- // OpenAI/Voyage API format
1085		- $request_body = [
1086		- 'input' => $text,
1087		- 'model' => $selected_model
1088		- ];
1089		-
1090		- // Add output_dimension for voyage-3-large
1091		- if ($selected_model === 'voyage-3-large') {
1092		- $request_body['output_dimension'] = 2048;
1093		- }
1094		-
1095		- // Prepare headers for OpenAI/Voyage
1096		- $headers = [
1097		- 'Content-Type' => 'application/json',
1098		- 'Authorization' => 'Bearer ' . $api_key
1099		- ];
1100		- }
1101		-
1102		- $args = [
1103		- 'body' => wp_json_encode($request_body),
1104		- 'headers' => $headers,
1105		- 'timeout' => 60,
1106		- 'redirection' => 5,
1107		- 'blocking' => true,
1108		- 'httpversion' => '1.0',
1109		- 'sslverify' => true,
1110		- ];
1111		-
1112		- $response = wp_remote_post($endpoint, $args);
1113		-
1114		- if (is_wp_error($response)) {
1115		- $message = 'Embedding request failed (connection): ' . $response->get_error_message();
1116		- if (class_exists('MxChat_Admin')) {
1117		- MxChat_Admin::mxchat_log_debug('embedding_error', $message, array('model' => $selected_model, 'bot_id' => $bot_id));
1118		- }
1119		- return new WP_Error('embedding_failed', $message, array(
1120		- 'branch' => 'cloud',
1121		- 'kind' => 'connection',
1122		- 'reason' => $response->get_error_message(),
1123		- 'model' => $selected_model,
1124		- ));
1125		- }
1126		-
1127		- $response_body = json_decode(wp_remote_retrieve_body($response), true);
1128		-
1129		- // Handle different response formats based on provider
1130		- if (strpos($selected_model, 'gemini-embedding') === 0) {
1131		- // Gemini API response format
1132		- if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
1133		- self::stamp_active_embedding_model($selected_model);
1134		- return $response_body['embedding']['values'];
1135		- } else {
1136		- return self::embedding_failure_error($response, $selected_model, $api_key, $bot_id);
1137		- }
1138		- } else {
1139		- // OpenAI/Voyage API response format
1140		- if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
1141		- self::stamp_active_embedding_model($selected_model);
1142		- return $response_body['data'][0]['embedding'];
1143		- } else {
1144		- return self::embedding_failure_error($response, $selected_model, $api_key, $bot_id);
1145		- }
1146		- }
1147		-}
1148		-
1149		-/**
1150		- * Build a WP_Error carrying the embedding provider's REAL failure reason,
1151		- * and record it in the Debug Mode log. Previously every failure path
1152		- * returned bare null, so customers saw only "Failed to generate embedding
1153		- * for content" / "Failed to store any chunks" with no cause (plan 4a7c0a).
1154		- *
1155		- * The API key never appears in provider response bodies (it travels in the
1156		- * request headers), but the reason is scrubbed for it anyway before it can
1157		- * reach a notice or the debug log.
1158		- */
1159		-private static function embedding_failure_error($response, $selected_model, $api_key, $bot_id) {
1160		- $status = (int) wp_remote_retrieve_response_code($response);
1161		- $raw = (string) wp_remote_retrieve_body($response);
1162		- $decoded = json_decode($raw, true);
1163		-
1164		- // Provider error shapes: OpenAI + Gemini use {"error":{"message":…}};
1165		- // Voyage uses {"detail":…}.
1166		- $reason = '';
1167		- if (is_array($decoded)) {
1168		- if (isset($decoded['error']['message']) && is_string($decoded['error']['message'])) {
1169		- $reason = $decoded['error']['message'];
1170		- } elseif (isset($decoded['detail']) && is_string($decoded['detail'])) {
1171		- $reason = $decoded['detail'];
1172		- }
1173		- }
1174		- if ($reason === '') {
1175		- $reason = ($raw !== '') ? substr($raw, 0, 200) : 'empty or malformed response';
1176		- }
1177		- if (is_string($api_key) && $api_key !== '') {
1178		- $reason = str_replace($api_key, '[redacted]', $reason);
1179		- }
1180		- $reason = substr($reason, 0, 300);
1181		- $message = sprintf('Embedding failed (%s, HTTP %d): %s', $selected_model, $status, $reason);
1182		-
1183		- if (class_exists('MxChat_Admin')) {
1184		- MxChat_Admin::mxchat_log_debug('embedding_error', $message, array(
1185		- 'model' => $selected_model,
1186		- 'status' => $status,
1187		- 'bot_id' => $bot_id,
1188		- ));
1189		- }
1190		-
1191		- // Structured data so the Integrator's query-side adapter can rebuild its
1192		- // typed error contract (auth/rate-limit/quota/invalid-response) without a
1193		- // second transport implementation (876edb). Additive — message unchanged.
1194		- return new WP_Error('embedding_failed', $message, array(
1195		- 'branch' => 'cloud',
1196		- 'status' => $status,
1197		- 'error_type' => (is_array($decoded) && isset($decoded['error']['type']) && is_string($decoded['error']['type'])) ? $decoded['error']['type'] : '',
1198		- 'reason' => $reason,
1199		- 'model' => $selected_model,
1200		- ));
1201		-}
1202		-
1203		-/**
1204		- * Generate an embedding via a Custom (OpenAI-compatible) provider's /embeddings route.
1205		- * Shared by every embedding entry point so the KNOWLEDGE-BASE INDEX side and the
1206		- * QUERY side route through the same model when the opt-in
1207		- * 'custom_provider_for_embeddings' setting is on. Mirrors the query-path logic in
1208		- * MxChat_Integrator::mxchat_generate_embedding_custom() but takes an explicit
1209		- * $options array so it is callable statically from utils + knowledge-manager.
1210		- *
1211		- * Returns a numeric array (the embedding vector) on success, or a human-readable
1212		- * error string on failure (so callers expecting a string error, like the
1213		- * knowledge-manager, can surface it directly; callers expecting array|null wrap it).
1214		- *
1215		- * @param string $text Text to embed.
1216		- * @param array $options The resolved mxchat options (must contain the custom_provider_* keys).
1217		- * @return array|string Embedding vector on success; error string on failure.
1218		- */
1219		-public static function generate_embedding_custom($text, $options) {
1220		- if (empty($text)) {
1221		- return 'No text provided for embedding generation';
1222		- }
1223		-
1224		- $base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
1225		- if (empty($base_url)) {
1226		- return 'Custom provider Base URL is not configured.';
1227		- }
1228		-
1229		- $api_key = isset($options['custom_provider_api_key']) ? trim((string) $options['custom_provider_api_key']) : '';
1230		- $auth_scheme = isset($options['custom_provider_auth_scheme']) ? $options['custom_provider_auth_scheme'] : 'bearer';
1231		- $api_version = isset($options['custom_provider_api_version']) ? trim((string) $options['custom_provider_api_version']) : '';
1232		-
1233		- // Embedding model: shared resolver (dedicated embedding model -> chat model
1234		- // -> 'default') — the mismatch warning's "selected" side reads the same chain.
1235		- $model = self::resolve_custom_embedding_model($options);
1236		-
1237		- $embed_url = $base_url . '/embeddings';
1238		- if (!empty($api_version)) {
1239		- $embed_url .= (strpos($embed_url, '?') === false ? '?' : '&') . 'api-version=' . rawurlencode($api_version);
1240		- }
1241		-
1242		- $headers = ['Content-Type' => 'application/json'];
1243		- if (!empty($api_key)) {
1244		- if ($auth_scheme === 'api-key') {
1245		- $headers['api-key'] = $api_key;
1246		- } else {
1247		- $headers['Authorization'] = 'Bearer ' . $api_key;
1248		- }
1249		- }
1250		-
1251		- $response = wp_remote_post($embed_url, [
1252		- 'headers' => $headers,
1253		- 'body' => wp_json_encode(['input' => $text, 'model' => $model]),
1254		- 'timeout' => 60,
1255		- ]);
1256		- if (is_wp_error($response)) {
1257		- return self::log_custom_embedding_failure(
1258		- 'Connection error when generating embeddings (custom provider): ' . $response->get_error_message(),
1259		- $model,
1260		- $api_key
1261		- );
1262		- }
1263		-
1264		- $status = wp_remote_retrieve_response_code($response);
1265		- $body = json_decode(wp_remote_retrieve_body($response), true);
1266		- if ($status !== 200) {
1267		- $msg = isset($body['error']['message']) ? $body['error']['message'] : 'HTTP ' . $status;
1268		- return self::log_custom_embedding_failure(
1269		- 'Custom embedding endpoint error: ' . $msg,
1270		- $model,
1271		- $api_key,
1272		- (int) $status
1273		- );
1274		- }
1275		- if (isset($body['data'][0]['embedding']) && is_array($body['data'][0]['embedding'])) {
1276		- // Stamp the custom model identity so the active-embedding-model mismatch
1277		- // warning reflects the real (custom) model rather than the built-in setting.
1278		- self::stamp_active_embedding_model('custom:' . $model);
1279		- return $body['data'][0]['embedding'];
1280		- }
1281		- return self::log_custom_embedding_failure('Invalid embedding response from custom provider.', $model, $api_key);
1282		-}
1283		-
1284		-/**
1285		- * Record a custom-provider embedding failure in the Debug Mode log, then
1286		- * return the message unchanged so callers keep their string-error contract.
1287		- * The cloud branch has logged its failures since 4a7c0a; the custom branch
1288		- * never did, so chat-side failures on Custom-provider installs were
1289		- * invisible to Debug Mode despite the 3.2.18 readme saying otherwise
1290		- * (plan 71e4b6). Same scrub-then-log shape as embedding_failure_error().
1291		- *
1292		- * @param string $message Human-readable failure (the caller's return value).
1293		- * @param string $model Resolved custom embedding model.
1294		- * @param string $api_key Scrubbed out of the logged message if it ever appears.
1295		- * @param int $status HTTP status when one was received, 0 otherwise.
1296		- * @return string The (scrubbed) message.
1297		- */
1298		-private static function log_custom_embedding_failure($message, $model, $api_key, $status = 0) {
1299		- if (is_string($api_key) && $api_key !== '') {
1300		- $message = str_replace($api_key, '[redacted]', $message);
1301		- }
1302		-
1303		- if (class_exists('MxChat_Admin')) {
1304		- $context = array('model' => 'custom:' . $model);
1305		- if ($status > 0) {
1306		- $context['status'] = $status;
1307		- }
1308		- MxChat_Admin::mxchat_log_debug('embedding_error', $message, $context);
1309		- }
1310		-
1311		- return $message;
1312		-}
1313		-
1314		-/**
1315		- * Submit content as multiple chunks
1316		- *
1317		- * Splits large content into chunks, generates embeddings for each,
1318		- * and stores them with chunk metadata for later reassembly.
1319		- *
1320		- * @param string $content The content to chunk and store
1321		- * @param string $source_url The source URL
1322		- * @param string $api_key The API key for embeddings
1323		- * @param string $bot_id The bot ID
1324		- * @param string $content_type The content type
1325		- * @param MxChat_Chunker $chunker The chunker instance
1326		- * @return bool|WP_Error True on success, WP_Error on failure
1327		- */
1328		-private static function submit_chunked_content($content, $source_url, $api_key, $bot_id, $content_type, $chunker) {
1329		- global $wpdb;
1330		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1331		-
1332		- //error_log('[MXCHAT-CHUNK-DEBUG] Starting chunked submission for: ' . $source_url);
1333		- //error_log('[MXCHAT-CHUNK-DEBUG] Content length: ' . strlen($content) . ' chars');
1334		-
1335		- // URL-less (manual) content needs a minted identity BEFORE chunk ids are derived:
1336		- // every chunk id is md5(source_url)_chunk_N, so with source_url = '' EVERY long
1337		- // manual document shared the md5('') prefix — and the clean-slate delete below
1338		- // wiped the PREVIOUS manual entry's chunks each time a new one was added. The
1339		- // single-vector paths already mint (mxchat:// in WP, manual_* in Pinecone); this
1340		- // was the one storage path that didn't. Keep an mxchat:// identity if the caller
1341		- // already carries one.
1342		- if (strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, 'upload://') !== 0 && !preg_match('#^https?://#i', $source_url)) {
1343		- $source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
1344		- }
1345		-
1346		- // First, delete any existing chunks for this URL (clean slate).
1347		- // Mirror-suspended: this is a re-store, not an entry removal — the Vector
1348		- // Store file is replaced (or kept, on hash match) by the caller's
1349		- // sync_upsert_entry after storage succeeds.
1350		- self::$vectorstore_mirror_suspended = true;
1351		- $delete_result = self::delete_chunks_for_url($source_url, $bot_id);
1352		- self::$vectorstore_mirror_suspended = false;
1353		- if (is_wp_error($delete_result)) {
1354		- //error_log('[MXCHAT-CHUNK-DEBUG] Warning: Failed to delete existing chunks: ' . $delete_result->get_error_message());
1355		- // Continue anyway - we'll overwrite with upsert
1356		- }
1357		-
1358		- // Split content into chunks
1359		- $chunks = $chunker->chunk_text($content);
1360		- $total_chunks = count($chunks);
1361		-
1362		- //error_log('[MXCHAT-CHUNK-DEBUG] Created ' . $total_chunks . ' chunks');
1363		- foreach ($chunks as $i => $chunk) {
1364		- //error_log('[MXCHAT-CHUNK-DEBUG] Chunk ' . $i . ' length: ' . strlen($chunk) . ' chars, preview: ' . substr($chunk, 0, 100));
1365		- }
1366		-
1367		- //error_log('[MXCHAT-CHUNK] Split content into ' . $total_chunks . ' chunks');
1368		-
1369		- if ($total_chunks === 0) {
1370		- return new WP_Error('chunking_failed', 'Content could not be split into chunks');
1371		- }
1372		-
1373		- $errors = array();
1374		- $embed_failures = 0;
1375		- $first_embed_reason = '';
1376		- $first_store_reason = '';
1377		- $is_pinecone = self::is_pinecone_enabled_for_bot($bot_id);
1378		-
1379		- foreach ($chunks as $index => $chunk_text) {
1380		- // Generate chunk metadata
1381		- $chunk_metadata = MxChat_Chunker::create_chunk_metadata($index, $total_chunks, $source_url);
1382		-
1383		- // AI-Engine-style aliases so external consumers (Pinecone/Qdrant/Chroma) can rely on
1384		- // a stable shorthand ('source'/'part_index'/'part_total') without parsing our internal names.
1385		- $chunk_metadata['source'] = $source_url;
1386		- $chunk_metadata['part_index'] = (int) $index;
1387		- $chunk_metadata['part_total'] = (int) $total_chunks;
1388		-
1389		- /**
1390		- * Filter the per-chunk metadata blob before it's written to the KB store.
1391		- *
1392		- * @param array $chunk_metadata Metadata array (source, part_index, part_total, chunk_index, total_chunks, source_url, parent_url_hash, document_type, ...).
1393		- * @param string $chunk_text The chunk text being stored.
1394		- * @param array $context ['bot_id' => string, 'content_type' => string, 'source_url' => string, 'part_index' => int, 'part_total' => int]
1395		- * @return array Updated metadata array.
1396		- */
1397		- $chunk_metadata = apply_filters(
1398		- 'mxchat_embedding_chunk_metadata',
1399		- $chunk_metadata,
1400		- $chunk_text,
1401		- array(
1402		- 'bot_id' => $bot_id,
1403		- 'content_type' => $content_type,
1404		- 'source_url' => $source_url,
1405		- 'part_index' => (int) $index,
1406		- 'part_total' => (int) $total_chunks,
1407		- )
1408		- );
1409		-
1410		- $chunk_vector_id = MxChat_Chunker::generate_chunk_vector_id($source_url, $index);
1411		-
1412		- //error_log('[MXCHAT-CHUNK] Processing chunk ' . ($index + 1) . '/' . $total_chunks . ' (ID: ' . $chunk_vector_id . ')');
1413		-
1414		- // Generate embedding for this chunk
1415		- $embedding_vector = self::generate_embedding($chunk_text, $api_key, $bot_id);
1416		-
1417		- if (!is_array($embedding_vector)) {
1418		- // Track embedding failures separately from storage failures, and
1419		- // keep the first provider reason seen — the two failure classes
1420		- // have opposite remedies (API key vs Pinecone/DB) (plan 4a7c0a).
1421		- $embed_failures++;
1422		- $reason = is_wp_error($embedding_vector) ? $embedding_vector->get_error_message() : '';
1423		- if ($reason !== '' && $first_embed_reason === '') {
1424		- $first_embed_reason = $reason;
1425		- }
1426		- $errors[] = new WP_Error('embedding_failed', 'Failed to generate embedding for chunk ' . $index . ($reason !== '' ? ' — ' . $reason : ''));
1427		- continue;
1428		- }
1429		-
1430		- if ($is_pinecone) {
1431		- // Store in Pinecone with chunk metadata
1432		- $result = self::store_chunk_in_pinecone(
1433		- $embedding_vector,
1434		- $chunk_text,
1435		- $source_url,
1436		- $chunk_vector_id,
1437		- $bot_id,
1438		- $content_type,
1439		- $chunk_metadata
1440		- );
1441		- } else {
1442		- // Store in WordPress DB with chunk metadata
1443		- $content_with_metadata = MxChat_Chunker::format_chunk_for_storage($chunk_text, $chunk_metadata);
1444		- $embedding_vector_serialized = maybe_serialize($embedding_vector);
1445		-
1446		- $result = self::store_chunk_in_wordpress_db(
1447		- $content_with_metadata,
1448		- $source_url,
1449		- $embedding_vector_serialized,
1450		- $table_name,
1451		- $content_type,
1452		- $chunk_metadata
1453		- );
1454		- }
1455		-
1456		- if (is_wp_error($result)) {
1457		- $errors[] = $result;
1458		- if ($first_store_reason === '') {
1459		- $first_store_reason = $result->get_error_message();
1460		- }
1461		- }
1462		- }
1463		-
1464		- if (count($errors) === $total_chunks) {
1465		- // Say WHICH stage failed — "failed to store" used to cover pure
1466		- // embedding failures too, sending customers to debug Pinecone when
1467		- // the problem was their embedding API key (plan 4a7c0a).
1468		- if ($embed_failures === $total_chunks) {
1469		- return new WP_Error('chunking_failed',
1470		- 'Failed to store any chunks — every chunk failed to embed'
1471		- . ($first_embed_reason !== '' ? ': ' . $first_embed_reason : '')
1472		- . ' Check the embedding provider API key and model under MxChat Settings.');
1473		- }
1474		- if ($embed_failures === 0) {
1475		- return new WP_Error('chunking_failed',
1476		- 'Failed to store any chunks — embeddings generated but storage failed'
1477		- . ($first_store_reason !== '' ? ': ' . $first_store_reason : '')
1478		- . ' Check the knowledge base storage (Pinecone index or database).');
1479		- }
1480		- return new WP_Error('chunking_failed', sprintf(
1481		- 'Failed to store any chunks — %d failed to embed%s and %d failed to store%s',
1482		- $embed_failures,
1483		- $first_embed_reason !== '' ? ' (' . $first_embed_reason . ')' : '',
1484		- $total_chunks - $embed_failures,
1485		- $first_store_reason !== '' ? ' (' . $first_store_reason . ')' : ''
1486		- ));
1487		- }
1488		-
1489		- if (!empty($errors)) {
1490		- $detail = $first_embed_reason !== '' ? $first_embed_reason : $first_store_reason;
1491		- return new WP_Error('chunking_partial_failure',
1492		- sprintf('Failed to store %d of %d chunks', count($errors), $total_chunks)
1493		- . ($detail !== '' ? ' — first error: ' . $detail : ''));
1494		- }
1495		-
1496		- //error_log('[MXCHAT-CHUNK] Successfully stored all ' . $total_chunks . ' chunks');
1497		- return true;
1498		-}
1499		-
1500		-/**
1501		- * Store a single chunk in Pinecone with chunk-specific metadata
1502		- */
1503		-private static function store_chunk_in_pinecone($embedding_vector, $chunk_text, $source_url, $vector_id, $bot_id, $content_type, $chunk_metadata) {
1504		- // Get Pinecone configuration
1505		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
1506		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
1507		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1508		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1509		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1510		- } else {
1511		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1512		- if (empty($bot_pinecone_config)) {
1513		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
1514		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1515		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1516		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1517		- } else {
1518		- $api_key = $bot_pinecone_config['api_key'] ?? '';
1519		- $host = $bot_pinecone_config['host'] ?? '';
1520		- $namespace = $bot_pinecone_config['namespace'] ?? '';
1521		- }
1522		- }
1523		-
1524		- if (empty($host) || empty($api_key)) {
1525		- return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
1526		- }
1527		-
1528		- $api_endpoint = "https://{$host}/vectors/upsert";
1529		-
1530		- // Build metadata with chunk information
1531		- $metadata = array(
1532		- 'text' => $chunk_text,
1533		- 'source_url' => $source_url,
1534		- 'type' => $content_type,
1535		- 'is_chunked' => true,
1536		- 'chunk_index' => $chunk_metadata['chunk_index'],
1537		- 'total_chunks' => $chunk_metadata['total_chunks'],
1538		- 'parent_url_hash' => $chunk_metadata['parent_url_hash'],
1539		- 'last_updated' => time(),
1540		- 'created_at' => time(),
1541		- 'bot_id' => $bot_id,
1542		- );
1543		-
1544		- $vector_data = array(
1545		- 'id' => $vector_id,
1546		- 'values' => $embedding_vector,
1547		- 'metadata' => $metadata
1548		- );
1549		-
1550		- $request_body = array(
1551		- 'vectors' => array($vector_data)
1552		- );
1553		-
1554		- if (!empty($namespace)) {
1555		- $request_body['namespace'] = $namespace;
1556		- }
1557		-
1558		- $response = wp_remote_post($api_endpoint, array(
1559		- 'headers' => array(
1560		- 'Api-Key' => $api_key,
1561		- 'accept' => 'application/json',
1562		- 'content-type' => 'application/json'
1563		- ),
1564		- 'body' => wp_json_encode($request_body),
1565		- 'timeout' => 30
1566		- ));
1567		-
1568		- if (is_wp_error($response)) {
1569		- return $response;
1570		- }
1571		-
1572		- $response_code = wp_remote_retrieve_response_code($response);
1573		- if ($response_code !== 200) {
1574		- return new WP_Error('pinecone_api', 'Pinecone API error: HTTP ' . $response_code);
1575		- }
1576		-
1577		- return true;
1578		-}
1579		-
1580		-/**
1581		- * Store a single chunk in WordPress database
1582		- */
1583		-private static function store_chunk_in_wordpress_db($content_with_metadata, $source_url, $embedding_vector_serialized, $table_name, $content_type, $chunk_metadata) {
1584		- global $wpdb;
1585		-
1586		- // For chunks, we always insert new rows (no duplicate checking)
1587		- // The URL includes chunk info in the metadata, but source_url stays the same for grouping
1588		- $result = $wpdb->insert(
1589		- $table_name,
1590		- array(
1591		- 'url' => $source_url,
1592		- 'article_content' => $content_with_metadata,
1593		- 'embedding_vector' => $embedding_vector_serialized,
1594		- 'source_url' => $source_url,
1595		- 'content_type' => $content_type,
1596		- 'timestamp' => current_time('mysql')
1597		- ),
1598		- array('%s', '%s', '%s', '%s', '%s', '%s')
1599		- );
1600		-
1601		- if ($result === false) {
1602		- return new WP_Error('database_failed', 'Failed to insert chunk: ' . $wpdb->last_error);
1603		- }
1604		-
1605		- return true;
1606		-}
1607		-
1608		-/**
1609		- * Delete all chunks for a given URL
1610		- *
1611		- * @param string $source_url The source URL
1612		- * @param string $bot_id The bot ID
1613		- * @return bool|WP_Error True on success, WP_Error on failure
1614		- */
1615		-public static function delete_chunks_for_url($source_url, $bot_id = 'default') {
1616		- //error_log('[MXCHAT-CHUNK-DELETE] Deleting chunks for URL: ' . $source_url);
1617		-
1618		- // Entry removal — mirror it to the Vector Store (unless a storage routine
1619		- // is mid-re-store, see $vectorstore_mirror_suspended).
1620		- if (!self::$vectorstore_mirror_suspended && class_exists('MxChat_Vectorstore_Manager')) {
1621		- MxChat_Vectorstore_Manager::sync_delete_entry($source_url, $bot_id);
1622		- }
1623		-
1624		- if (self::is_pinecone_enabled_for_bot($bot_id)) {
1625		- return self::delete_pinecone_chunks_by_url($source_url, $bot_id);
1626		- } else {
1627		- return self::delete_wordpress_chunks_by_url($source_url);
1628		- }
1629		-}
1630		-
1631		-/**
1632		- * Delete all chunks for a URL from Pinecone
1633		- */
1634		-/**
1635		- * Delete leftover md5(url)_chunk_N vectors after a URL's content was re-stored
1636		- * as a SINGLE vector (content shrank below the chunk threshold on edit/re-import).
1637		- * Unlike delete_pinecone_chunks_by_url this leaves the base id alone — the caller
1638		- * just upserted the new content there. Failure is logged, not fatal: the save
1639		- * itself succeeded, and the next save retries the sweep.
1640		- */
1641		-private static function cleanup_pinecone_chunk_stragglers($source_url, $bot_id) {
1642		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
1643		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
1644		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1645		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1646		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1647		- } else {
1648		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1649		- if (empty($bot_pinecone_config)) {
1650		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
1651		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1652		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1653		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1654		- } else {
1655		- $api_key = $bot_pinecone_config['api_key'] ?? '';
1656		- $host = $bot_pinecone_config['host'] ?? '';
1657		- $namespace = $bot_pinecone_config['namespace'] ?? '';
1658		- }
1659		- }
1660		-
1661		- if (empty($host) || empty($api_key)) {
1662		- return;
1663		- }
1664		-
1665		- $stragglers = array();
1666		-
1667		- // Pinecone /vectors/list is a GET endpoint with query-string parameters (a POST
1668		- // answers 200-with-an-empty-body, which reads as "no stragglers").
1669		- $query_params = array(
1670		- 'prefix' => md5($source_url) . '_chunk_',
1671		- 'limit' => 100,
1672		- );
1673		- if (!empty($namespace)) {
1674		- $query_params['namespace'] = $namespace;
1675		- }
1676		-
1677		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
1678		-
1679		- do {
1680		- $list_response = wp_remote_get($list_url, array(
1681		- 'headers' => array(
1682		- 'Api-Key' => $api_key,
1683		- 'accept' => 'application/json',
1684		- ),
1685		- 'timeout' => 30,
1686		- ));
1687		-
1688		- if (is_wp_error($list_response) || wp_remote_retrieve_response_code($list_response) !== 200) {
1689		- break;
1690		- }
1691		-
1692		- $list_data = json_decode(wp_remote_retrieve_body($list_response), true);
1693		- if (!empty($list_data['vectors'])) {
1694		- foreach ($list_data['vectors'] as $vector) {
1695		- if (isset($vector['id'])) {
1696		- $stragglers[] = $vector['id'];
1697		- }
1698		- }
1699		- }
1700		-
1701		- $next_token = $list_data['pagination']['next'] ?? '';
1702		- if (empty($next_token)) {
1703		- break;
1704		- }
1705		-
1706		- $query_params['paginationToken'] = $next_token;
1707		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
1708		- } while (true);
1709		-
1710		- if (empty($stragglers)) {
1711		- return;
1712		- }
1713		-
1714		- $delete_body = array('ids' => $stragglers);
1715		- if (!empty($namespace)) {
1716		- $delete_body['namespace'] = $namespace;
1717		- }
1718		-
1719		- $delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
1720		- 'headers' => array(
1721		- 'Api-Key' => $api_key,
1722		- 'accept' => 'application/json',
1723		- 'content-type' => 'application/json'
1724		- ),
1725		- 'body' => wp_json_encode($delete_body),
1726		- 'timeout' => 30
1727		- ));
1728		-
1729		- if ((is_wp_error($delete_response) || wp_remote_retrieve_response_code($delete_response) !== 200)
1730		- && class_exists('MxChat_Admin') && method_exists('MxChat_Admin', 'mxchat_log_debug')) {
1731		- MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to sweep stale chunk vectors after single-vector re-store', array('source_url' => $source_url, 'bot_id' => $bot_id, 'count' => count($stragglers)));
1732		- }
1733		-}
1734		-
1735		-private static function delete_pinecone_chunks_by_url($source_url, $bot_id) {
1736		- // Get Pinecone configuration
1737		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
1738		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
1739		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1740		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1741		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1742		- } else {
1743		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1744		- if (empty($bot_pinecone_config)) {
1745		- $pinecone_options = get_option('mxchat_pinecone_addon_options');
1746		- $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1747		- $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1748		- $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1749		- } else {
1750		- $api_key = $bot_pinecone_config['api_key'] ?? '';
1751		- $host = $bot_pinecone_config['host'] ?? '';
1752		- $namespace = $bot_pinecone_config['namespace'] ?? '';
1753		- }
1754		- }
1755		-
1756		- if (empty($host) || empty($api_key)) {
1757		- return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
1758		- }
1759		-
1760		- $base_vector_id = md5($source_url);
1761		- $vectors_to_delete = array();
1762		-
1763		- // Add the original single-vector ID (for non-chunked content)
1764		- $vectors_to_delete[] = $base_vector_id;
1765		-
1766		- // Pinecone /vectors/list is a GET endpoint with query-string parameters; a POST here returns a
1767		- // non-200 silently and we end up only deleting the base vector, leaving chunks orphaned.
1768		- $query_params = array(
1769		- 'prefix' => $base_vector_id . '_chunk_',
1770		- 'limit' => 100,
1771		- );
1772		- if (!empty($namespace)) {
1773		- $query_params['namespace'] = $namespace;
1774		- }
1775		-
1776		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
1777		-
1778		- // Paginate in case a URL has more than 100 chunks.
1779		- do {
1780		- $list_response = wp_remote_get($list_url, array(
1781		- 'headers' => array(
1782		- 'Api-Key' => $api_key,
1783		- 'accept' => 'application/json',
1784		- ),
1785		- 'timeout' => 30,
1786		- ));
1787		-
1788		- if (is_wp_error($list_response) || wp_remote_retrieve_response_code($list_response) !== 200) {
1789		- break;
1790		- }
1791		-
1792		- $list_data = json_decode(wp_remote_retrieve_body($list_response), true);
1793		- if (!empty($list_data['vectors'])) {
1794		- foreach ($list_data['vectors'] as $vector) {
1795		- if (isset($vector['id'])) {
1796		- $vectors_to_delete[] = $vector['id'];
1797		- }
1798		- }
1799		- }
1800		-
1801		- $next_token = $list_data['pagination']['next'] ?? '';
1802		- if (empty($next_token)) {
1803		- break;
1804		- }
1805		-
1806		- $query_params['paginationToken'] = $next_token;
1807		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
1808		- } while (true);
1809		-
1810		- if (empty($vectors_to_delete)) {
1811		- //error_log('[MXCHAT-CHUNK-DELETE] No vectors found to delete');
1812		- return true;
1813		- }
1814		-
1815		- //error_log('[MXCHAT-CHUNK-DELETE] Deleting ' . count($vectors_to_delete) . ' vectors from Pinecone');
1816		-
1817		- // Delete vectors
1818		- $delete_url = "https://{$host}/vectors/delete";
1819		-
1820		- $delete_body = array(
1821		- 'ids' => $vectors_to_delete
1822		- );
1823		-
1824		- if (!empty($namespace)) {
1825		- $delete_body['namespace'] = $namespace;
1826		- }
1827		-
1828		- $delete_response = wp_remote_post($delete_url, array(
1829		- 'headers' => array(
1830		- 'Api-Key' => $api_key,
1831		- 'accept' => 'application/json',
1832		- 'content-type' => 'application/json'
1833		- ),
1834		- 'body' => wp_json_encode($delete_body),
1835		- 'timeout' => 30
1836		- ));
1837		-
1838		- if (is_wp_error($delete_response)) {
1839		- return $delete_response;
1840		- }
1841		-
1842		- $response_code = wp_remote_retrieve_response_code($delete_response);
1843		- if ($response_code !== 200) {
1844		- return new WP_Error('pinecone_delete', 'Failed to delete vectors: HTTP ' . $response_code);
1845		- }
1846		-
1847		- return true;
1848		-}
1849		-
1850		-/**
1851		- * Delete all chunks for a URL from WordPress database
1852		- */
1853		-private static function delete_wordpress_chunks_by_url($source_url) {
1854		- global $wpdb;
1855		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1856		-
1857		- // Delete all rows with this source_url (handles both chunked and non-chunked)
1858		- $result = $wpdb->delete(
1859		- $table_name,
1860		- array('source_url' => $source_url),
1861		- array('%s')
1862		- );
1863		-
1864		- if ($result === false) {
1865		- return new WP_Error('database_delete', 'Failed to delete chunks: ' . $wpdb->last_error);
1866		- }
1867		-
1868		- //error_log('[MXCHAT-CHUNK-DELETE] Deleted ' . $result . ' rows from WordPress DB');
1869		- return true;
1870		-}
1871		-
1872		-/**
1873		- * Hybrid keyword boost (plan-38ffa1): detect whether the WP-DB knowledge
1874		- * table can serve the keyword leg via a MySQL FULLTEXT index, creating the
1875		- * index if needed. Detection runs once and caches the answer in the
1876		- * mxchat_hybrid_keyword_capability option ('fulltext' | 'like'); pass
1877		- * $force to re-detect. LIKE is the graceful fallback for shared hosts
1878		- * whose ALTER fails — the feature works either way, FULLTEXT just ranks
1879		- * better and scales.
1880		- *
1881		- * @param bool $force Re-run detection even if a cached answer exists.
1882		- * @return string 'fulltext' or 'like'
1883		- */
1884		-public static function mxchat_hybrid_detect_capability($force = false) {
1885		- $cached = get_option('mxchat_hybrid_keyword_capability', '');
1886		- if (!$force && in_array($cached, array('fulltext', 'like'), true)) {
1887		- return $cached;
1888		- }
1889		-
1890		- global $wpdb;
1891		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
1892		-
1893		- $index_exists = $wpdb->get_var("SHOW INDEX FROM {$table} WHERE Key_name = 'mxchat_content_ft'");
1894		- if (!$index_exists) {
1895		- // Suppress the visible error on hosts where this is not permitted —
1896		- // failure is an expected, handled outcome (LIKE fallback).
1897		- $suppress = $wpdb->suppress_errors(true);
1898		- $wpdb->query("ALTER TABLE {$table} ADD FULLTEXT INDEX mxchat_content_ft (article_content)");
1899		- $wpdb->suppress_errors($suppress);
1900		- $index_exists = $wpdb->get_var("SHOW INDEX FROM {$table} WHERE Key_name = 'mxchat_content_ft'");
1901		- }
1902		-
1903		- $capability = $index_exists ? 'fulltext' : 'like';
1904		- update_option('mxchat_hybrid_keyword_capability', $capability);
1905		- return $capability;
1906		-}
1907		-
1908		-/**
1909		- * Public entry for re-embedding already-stored content in place (wp mxchat
1910		- * rtl-repair, plan d1e6f7). Thin wrapper so the repair CLI gets the exact
1911		- * provider routing the import path uses — the repaired vector must come from
1912		- * the same model family the bot indexes with, or retrieval stays broken.
1913		- */
1914		-public static function regenerate_embedding($text, $api_key, $bot_id = 'default') {
1915		- return self::generate_embedding($text, $api_key, $bot_id);
1916		-}
1917		-
1918		-/**
1919		- * Restore logical character order in PDF-extracted RTL text (plan 32bf9e).
1920		- *
1921		- * The bundled Smalot parser only un-reverses text runs tagged with the
1922		- * ReversedChars marked-content operator (Word emits it; LibreOffice and most
1923		- * other producers do not), so their Hebrew/Arabic PDFs extract in visual
1924		- * (reversed) order and embed/search as garbage. This is OUR post-processing
1925		- * seam over getText() — the parser itself is never patched (it gets replaced
1926		- * wholesale on library updates).
1927		- *
1928		- * Heuristic and deliberately conservative, per line:
1929		- * - lines without strong RTL codepoints are untouched (a fully-Latin line in
1930		- * an RTL document therefore stays as extracted — accepted limitation);
1931		- * - Arabic presentation forms are a definitive visual-order signal (they only
1932		- * appear in shaped output): de-shape to base letters and reverse;
1933		- * - otherwise flip only on positive evidence — Hebrew final-letter position
1934		- * (a sofit at word START only happens in reversed text) or sentence
1935		- * punctuation position (leading in visual order, trailing in logical);
1936		- * - ambiguous lines are left alone: a conservative miss beats corrupting a
1937		- * Word-produced extraction the parser already handled (the double-flip
1938		- * guard this plan's approval named mandatory).
1939		- *
1940		- * @param string $text One extracted page string, straight from getText().
1941		- * @param string $context Caller tag for the Debug Mode entry (site + page).
1942		- * @return string Text with RTL lines restored to logical order.
1943		- */
1944		-public static function normalize_pdf_rtl($text, $context = '') {
1945		- if (!is_string($text) || '' === $text) {
1946		- return $text;
1947		- }
1948		- // Escape hatch for sites whose PDFs already extract logically.
1949		- if (!apply_filters('mxchat_pdf_rtl_normalize', true, $text)) {
1950		- return $text;
1951		- }
1952		- // Fast bail: nothing RTL anywhere in the page.
1953		- if (!preg_match('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $text)) {
1954		- return $text;
1955		- }
1956		-
1957		- $parts = preg_split('/(\R)/u', $text, -1, PREG_SPLIT_DELIM_CAPTURE);
1958		- if (false === $parts) {
1959		- return $text;
1960		- }
1961		-
1962		- $flipped_lines = 0;
1963		- $deshaped_lines = 0;
1964		- foreach ($parts as $i => $part) {
1965		- if ('' === $part || preg_match('/^\R$/u', $part)) {
1966		- continue;
1967		- }
1968		- $was_flipped = false;
1969		- $was_deshaped = false;
1970		- $new = self::pdf_rtl_normalize_line($part, $was_flipped, $was_deshaped);
1971		- if ($new !== $part) {
1972		- $parts[$i] = $new;
1973		- }
1974		- if ($was_flipped) {
1975		- $flipped_lines++;
1976		- }
1977		- if ($was_deshaped) {
1978		- $deshaped_lines++;
1979		- }
1980		- }
1981		-
1982		- if (($flipped_lines || $deshaped_lines) && class_exists('MxChat_Admin')) {
1983		- MxChat_Admin::mxchat_log_debug('pdf_rtl_normalized', 'RTL PDF text restored to logical order', array(
1984		- 'context' => (string) $context,
1985		- 'lines_flipped' => $flipped_lines,
1986		- 'lines_deshaped' => $deshaped_lines,
1987		- 'decision' => 'visual-order extraction detected',
1988		- ));
1989		- }
1990		-
1991		- return implode('', $parts);
1992		-}
1993		-
1994		-/**
1995		- * Normalize one line. Sets $flipped/$deshaped for the caller's debug entry.
1996		- */
1997		-private static function pdf_rtl_normalize_line($line, &$flipped, &$deshaped) {
1998		- $flipped = false;
1999		- $deshaped = false;
2000		-
2001		- if (!preg_match('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $line)) {
2002		- return $line;
2003		- }
2004		-
2005		- $has_forms = (bool) preg_match('/[\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $line);
2006		- $work = $line;
2007		- if ($has_forms) {
2008		- $work = strtr($work, self::pdf_rtl_deshape_map());
2009		- $deshaped = ($work !== $line);
2010		- }
2011		-
2012		- $verdict = 'ambiguous';
2013		- if ($has_forms) {
2014		- // Shaped glyph codepoints only exist in visual-order output.
2015		- $verdict = 'visual';
2016		- } else {
2017		- // Strong-direction dominance gate first: an LTR-dominant line with an
2018		- // embedded RTL word is not flip material.
2019		- $rtl_count = preg_match_all('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}]/u', $work, $m_rtl);
2020		- $ltr_count = preg_match_all('/[A-Za-z]/u', $work, $m_ltr);
2021		- if ($rtl_count < 1 || $rtl_count <= $ltr_count) {
2022		- return $line;
2023		- }
2024		-
2025		- // Hebrew final letters (ך ם ן ף ץ) end words in logical text; one at
2026		- // a word START (Hebrew letter follows, none precedes) is reversal
2027		- // evidence. Positional, so it survives the line being reversed.
2028		- $sofit_initial = preg_match_all('/(?<![\x{05D0}-\x{05EA}])[\x{05DA}\x{05DD}\x{05DF}\x{05E3}\x{05E5}](?=[\x{05D0}-\x{05EA}])/u', $work, $m_i);
2029		- $sofit_terminal = preg_match_all('/(?<=[\x{05D0}-\x{05EA}])[\x{05DA}\x{05DD}\x{05DF}\x{05E3}\x{05E5}](?![\x{05D0}-\x{05EA}])/u', $work, $m_t);
2030		- if ($sofit_initial > $sofit_terminal) {
2031		- $verdict = 'visual';
2032		- } elseif ($sofit_terminal > $sofit_initial) {
2033		- $verdict = 'logical';
2034		- } else {
2035		- // Sentence punctuation lands at the visual LEFT edge of an RTL
2036		- // line, i.e. the START of a visual-order extraction.
2037		- $trimmed = trim($work);
2038		- $starts_punct = (bool) preg_match('/^[.?!:;,]/u', $trimmed);
2039		- $ends_punct = (bool) preg_match('/[.?!:;,]$/u', $trimmed);
2040		- if ($starts_punct && !$ends_punct) {
2041		- $verdict = 'visual';
2042		- } elseif ($ends_punct && !$starts_punct) {
2043		- $verdict = 'logical';
2044		- }
2045		- }
2046		- }
2047		-
2048		- if ('visual' !== $verdict) {
2049		- // Ambiguous or logical: hand back the original line UNLESS we
2050		- // de-shaped (de-shaping alone is always safe — same letters, same
2051		- // order, un-ligated).
2052		- return $deshaped ? $work : $line;
2053		- }
2054		-
2055		- $flipped = true;
2056		- return self::pdf_rtl_flip_line($work);
2057		-}
2058		-
2059		-/**
2060		- * Reverse a visual-order line into logical order: full character reversal,
2061		- * mirror paired punctuation, then re-reverse embedded LTR runs (Latin words
2062		- * and digit sequences, incl. Arabic-Indic digits) so they stay readable.
2063		- */
2064		-private static function pdf_rtl_flip_line($line) {
2065		- $chars = preg_split('//u', $line, -1, PREG_SPLIT_NO_EMPTY);
2066		- if (false === $chars) {
2067		- return $line;
2068		- }
2069		- $reversed = implode('', array_reverse($chars));
2070		- $reversed = strtr($reversed, array(
2071		- '(' => ')', ')' => '(',
2072		- '[' => ']', ']' => '[',
2073		- '{' => '}', '}' => '{',
2074		- '<' => '>', '>' => '<',
2075		- ));
2076		- $restored = preg_replace_callback(
2077		- '/[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9}](?:[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9} .,\'"%\-:\/]*[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9}])?/u',
2078		- function ($m) {
2079		- $run = preg_split('//u', $m[0], -1, PREG_SPLIT_NO_EMPTY);
2080		- return false === $run ? $m[0] : implode('', array_reverse($run));
2081		- },
2082		- $reversed
2083		- );
2084		- return null === $restored ? $reversed : $restored;
2085		-}
2086		-
2087		-/**
2088		- * Arabic presentation forms (A + B) -> base letters. Built once from range
2089		- * specs rather than ~120 hand-written literal entries; every codepoint in a
2090		- * range maps to the same base sequence (isolated/final/initial/medial forms
2091		- * of one letter are contiguous in the FE70 block).
2092		- */
2093		-private static function pdf_rtl_deshape_map() {
2094		- static $map = null;
2095		- if (null !== $map) {
2096		- return $map;
2097		- }
2098		- $ranges = array(
2099		- // Form B harakat (each pair = standalone + tatweel-joined form).
2100		- array(0xFE70, 0xFE71, array(0x064B)), array(0xFE72, 0xFE72, array(0x064C)),
2101		- array(0xFE74, 0xFE74, array(0x064D)), array(0xFE76, 0xFE77, array(0x064E)),
2102		- array(0xFE78, 0xFE79, array(0x064F)), array(0xFE7A, 0xFE7B, array(0x0650)),
2103		- array(0xFE7C, 0xFE7D, array(0x0651)), array(0xFE7E, 0xFE7F, array(0x0652)),
2104		- // Form B letters.
2105		- array(0xFE80, 0xFE80, array(0x0621)), array(0xFE81, 0xFE82, array(0x0622)),
2106		- array(0xFE83, 0xFE84, array(0x0623)), array(0xFE85, 0xFE86, array(0x0624)),
2107		- array(0xFE87, 0xFE88, array(0x0625)), array(0xFE89, 0xFE8C, array(0x0626)),
2108		- array(0xFE8D, 0xFE8E, array(0x0627)), array(0xFE8F, 0xFE92, array(0x0628)),
2109		- array(0xFE93, 0xFE94, array(0x0629)), array(0xFE95, 0xFE98, array(0x062A)),
2110		- array(0xFE99, 0xFE9C, array(0x062B)), array(0xFE9D, 0xFEA0, array(0x062C)),
2111		- array(0xFEA1, 0xFEA4, array(0x062D)), array(0xFEA5, 0xFEA8, array(0x062E)),
2112		- array(0xFEA9, 0xFEAA, array(0x062F)), array(0xFEAB, 0xFEAC, array(0x0630)),
2113		- array(0xFEAD, 0xFEAE, array(0x0631)), array(0xFEAF, 0xFEB0, array(0x0632)),
2114		- array(0xFEB1, 0xFEB4, array(0x0633)), array(0xFEB5, 0xFEB8, array(0x0634)),
2115		- array(0xFEB9, 0xFEBC, array(0x0635)), array(0xFEBD, 0xFEC0, array(0x0636)),
2116		- array(0xFEC1, 0xFEC4, array(0x0637)), array(0xFEC5, 0xFEC8, array(0x0638)),
2117		- array(0xFEC9, 0xFECC, array(0x0639)), array(0xFECD, 0xFED0, array(0x063A)),
2118		- array(0xFED1, 0xFED4, array(0x0641)), array(0xFED5, 0xFED8, array(0x0642)),
2119		- array(0xFED9, 0xFEDC, array(0x0643)), array(0xFEDD, 0xFEE0, array(0x0644)),
2120		- array(0xFEE1, 0xFEE4, array(0x0645)), array(0xFEE5, 0xFEE8, array(0x0646)),
2121		- array(0xFEE9, 0xFEEC, array(0x0647)), array(0xFEED, 0xFEEE, array(0x0648)),
2122		- array(0xFEEF, 0xFEF0, array(0x0649)), array(0xFEF1, 0xFEF4, array(0x064A)),
2123		- // Form B lam-alef ligatures decompose to two letters.
2124		- array(0xFEF5, 0xFEF6, array(0x0644, 0x0622)), array(0xFEF7, 0xFEF8, array(0x0644, 0x0623)),
2125		- array(0xFEF9, 0xFEFA, array(0x0644, 0x0625)), array(0xFEFB, 0xFEFC, array(0x0644, 0x0627)),
2126		- // Form A: Persian / Urdu letters in common use.
2127		- array(0xFB56, 0xFB59, array(0x067E)), array(0xFB66, 0xFB69, array(0x0679)),
2128		- array(0xFB7A, 0xFB7D, array(0x0686)), array(0xFB88, 0xFB89, array(0x0688)),
2129		- array(0xFB8A, 0xFB8B, array(0x0698)), array(0xFB8E, 0xFB91, array(0x06A9)),
2130		- array(0xFB92, 0xFB95, array(0x06AF)), array(0xFBA6, 0xFBA9, array(0x06C1)),
2131		- array(0xFBAA, 0xFBAD, array(0x06BE)), array(0xFBAE, 0xFBAF, array(0x06D2)),
2132		- array(0xFBFC, 0xFBFF, array(0x06CC)),
2133		- );
2134		- $map = array();
2135		- foreach ($ranges as $range) {
2136		- $base = '';
2137		- foreach ($range[2] as $cp) {
2138		- $base .= self::pdf_rtl_cp_to_utf8($cp);
2139		- }
2140		- for ($cp = $range[0]; $cp <= $range[1]; $cp++) {
2141		- $map[self::pdf_rtl_cp_to_utf8($cp)] = $base;
2142		- }
2143		- }
2144		- return $map;
2145		-}
2146		-
2147		-/**
2148		- * Codepoint to UTF-8 without ext-intl / mbstring entity tricks (PHP 7.2 floor).
2149		- */
2150		-private static function pdf_rtl_cp_to_utf8($cp) {
2151		- if ($cp < 0x80) {
2152		- return chr($cp);
2153		- }
2154		- if ($cp < 0x800) {
2155		- return chr(0xC0 | ($cp >> 6)) . chr(0x80 | ($cp & 0x3F));
2156		- }
2157		- if ($cp < 0x10000) {
2158		- return chr(0xE0 | ($cp >> 12)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
2159		- }
2160		- return chr(0xF0 | ($cp >> 18)) . chr(0x80 | (($cp >> 12) & 0x3F)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
2161		-}
2162		-}
	1	+<?php
	2	+if (!defined('ABSPATH')) {
	3	+ exit; // Exit if accessed directly
	4	+}
	5	+
	6	+class MxChat_Utils {
	7	+
	8	+ /**
	9	+ * Submit content and its embedding to the database.
	10	+ *
	11	+ * @param string $content The content to be embedded.
	12	+ * @param string $source_url The source URL of the content.
	13	+ * @param string $api_key The API key used for generating embeddings.
	14	+ */
	15	+ public static function submit_content_to_db($content, $source_url, $api_key) {
	16	+ global $wpdb;
	17	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	18	+
	19	+ // Sanitize the source URL
	20	+ $source_url = esc_url_raw($source_url);
	21	+
	22	+ // Generate the embedding using the API key
	23	+ $embedding_vector = self::generate_embedding($content, $api_key);
	24	+
	25	+ if (is_array($embedding_vector)) {
	26	+ $embedding_vector_serialized = serialize($embedding_vector);
	27	+
	28	+ // Insert or update the record in the database
	29	+ $wpdb->replace(
	30	+ $table_name,
	31	+ array(
	32	+ 'article_content' => $content,
	33	+ 'embedding_vector' => $embedding_vector_serialized,
	34	+ 'source_url' => $source_url,
	35	+ ),
	36	+ array(
	37	+ '%s',
	38	+ '%s',
	39	+ '%s',
	40	+ )
	41	+ );
	42	+ } else {
	43	+ //error_log('Embedding generation failed for content from ' . $source_url);
	44	+ }
	45	+ }
	46	+
	47	+ /**
	48	+ * Generate an embedding for the given text.
	49	+ *
	50	+ * @param string $text The text to be embedded.
	51	+ * @param string $api_key The API key used for generating embeddings.
	52	+ * @return array|null The embedding vector or null on failure.
	53	+ */
	54	+ private static function generate_embedding($text, $api_key) {
	55	+ $endpoint = 'https://api.openai.com/v1/embeddings';
	56	+
	57	+ $body = wp_json_encode([
	58	+ 'input' => $text,
	59	+ 'model' => 'text-embedding-ada-002'
	60	+ ]);
	61	+
	62	+ $args = [
	63	+ 'body' => $body,
	64	+ 'headers' => [
	65	+ 'Content-Type' => 'application/json',
	66	+ 'Authorization' => 'Bearer ' . $api_key,
	67	+ ],
	68	+ 'timeout' => 60,
	69	+ 'redirection' => 5,
	70	+ 'blocking' => true,
	71	+ 'httpversion' => '1.0',
	72	+ 'sslverify' => true,
	73	+ ];
	74	+
	75	+ $response = wp_remote_post($endpoint, $args);
	76	+
	77	+ if (is_wp_error($response)) {
	78	+ //error_log('Error generating embedding: ' . $response->get_error_message());
	79	+ return null;
	80	+ }
	81	+
	82	+ $response_body = json_decode(wp_remote_retrieve_body($response), true);
	83	+
	84	+ if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
	85	+ return $response_body['data'][0]['embedding'];
	86	+ } else {
	87	+ //error_log('Invalid response received from embedding API.');
	88	+ return null;
	89	+ }
	90	+ }
	91	+}
	92	+
	93	+?>