mxchat-basic / includes / class-mxchat-utils.php

class-mxchat-utils.php in MxChat – AI Chatbot & Content Generation for WordPress 3.2.22, at includes/class-mxchat-utils.php

2,344 lines 99.2 KB

Raw Download Zip

1	<?php
2	if (!defined('ABSPATH')) {
3	exit; // Exit if accessed directly
4	}
5
6	class MxChat_Utils {
7
8	/**
9	* True while a storage routine is re-writing an entry it is about to re-add
10	* (submit_chunked_content's clean-slate delete). delete_chunks_for_url() skips
11	* the Vector Store mirror-delete while set — an internal re-store is not an
12	* entry removal, and mirroring it would delete-then-reupload the entry's file
13	* on every chunked save (plan 15b5c6).
14	*/
15	private static $vectorstore_mirror_suspended = false;
16
17	/**
18	* Validate a client-supplied session id (plan-mxchat-20260731-d42bec).
19	*
20	* sanitize_text_field() — which every session_id read site used before this —
21	* preserves '/' and '..'. Harmless where the value is only an option or
22	* transient key suffix, but mxchat_send_delayed_transcript() interpolates it
23	* into a filesystem path, so '../../../../path/x' wrote, emailed and deleted a
24	* file outside the uploads dir.
25	*
26	* REJECTS rather than rewrites: a silently-stripped id would orphan the
27	* conversation it belongs to, which is harder to diagnose than a clean refusal.
28	* Returns '' for anything malformed, so call sites fall into the empty-session
29	* error paths they already have.
30	*
31	* The generator only ever emits 'mxchat_chat_' + 32 hex chars
32	* (class-mxchat-integrator.php, js/chat-script.js), so this is not restrictive
33	* in practice. Length ceiling is deliberate — session ids are also used as
34	* option-name suffixes, and WP option names cap at 191 chars.
35	*
36	* @param mixed $raw Raw request value.
37	* @return string The id if well-formed, '' otherwise.
38	*/
39	public static function sanitize_session_id($raw) {
40	if (!is_scalar($raw)) {
41	return '';
42	}
43	$val = trim((string) $raw);
44	if ($val === '') {
45	return '';
46	}
47	return preg_match('/\A[A-Za-z0-9_-]{1,128}\z/', $val) ? $val : '';
48	}
49
50	/**
51	* Sanitize the lead-capture consent-checkbox label (plan b062c4).
52	*
53	* The label is owner-supplied and renders inside the widget's email form, so
54	* this is a security boundary, not a formatting nicety. One explicit
55	* allowlist, used at BOTH save time (options.php sanitize + autosave AJAX)
56	* and render time (widget form, admin surfaces) so the two can never drift:
57	* an anchor — the whole point is "I agree to the <a>Privacy Policy</a>" —
58	* plus inline emphasis. No block tags, no images, no style attributes.
59	*
60	* The stored consent record keeps this exact sanitized string as "the text
61	* the visitor saw", so it must be deterministic: same input, same output,
62	* whichever path ran it.
63	*
64	* @param mixed $raw Owner-entered label.
65	* @return string Sanitized label, capped at 1000 chars.
66	*/
67	public static function sanitize_consent_label($raw) {
68	if (!is_scalar($raw)) {
69	return '';
70	}
71
72	$allowed = array(
73	'a' => array(
74	'href' => true,
75	'title' => true,
76	'target' => true,
77	'rel' => true,
78	),
79	'strong' => array(),
80	'em' => array(),
81	'br' => array(),
82	);
83
84	$label = wp_kses(trim((string) $raw), $allowed);
85
86	return mb_substr($label, 0, 1000);
87	}
88
89	/**
90	* Per-request cache for get_session_history(). Mirrors get_option()'s
91	* request-scoped caching, which the mxchat_history_ option reads got for
92	* free before plan 839c4c moved history reads onto the transcripts table.
93	*/
94	private static $history_cache = array();
95
96	/**
97	* Session chat history read from the transcripts table, in the exact array
98	* shape the legacy mxchat_history_<sid> option stored (plan 839c4c). The
99	* option was a second copy of state the table already held — measured
100	* byte-identical in role/content/order on 174 of 177 real sessions, with
101	* the table a superset on the rest — at up to 64 KB per option row. The
102	* table is now the single store; nothing writes the option any more.
103	*
104	* Shape notes, load-bearing for the consumers:
105	* - id: the transcripts row id (int). Integer ids make the pollers'
106	* ">" comparisons correct where the old uniqid() strings only worked by
107	* accident of hex ordering.
108	* - timestamp: milliseconds, derived from the table's second-resolution GMT
109	* column (x1000). Consumers comparing against a real-millisecond client
110	* cutoff MUST floor the cutoff to the second and err inclusive — see the
111	* persistence-off filters in class-mxchat-integrator.php.
112	* - agent_name: the row's user_identifier, which the writer sets to the
113	* same displayed_name value the option carried (agent name when present,
114	* else email, else identifier).
115	*
116	* Public and static so mxchat-woo / mxchat-forms can call the same accessor
117	* as core, guarded with method_exists against an older mxchat-basic.
118	*
119	* @param string $session_id
120	* @return array[] Chronological entries: id, role, content, timestamp, agent_name.
121	*/
122	public static function get_session_history($session_id) {
123	global $wpdb;
124
125	$session_id = self::sanitize_session_id($session_id);
126	if ($session_id === '') {
127	return array();
128	}
129
130	if (array_key_exists($session_id, self::$history_cache)) {
131	return self::$history_cache[$session_id];
132	}
133
134	$table = $wpdb->prefix . 'mxchat_chat_transcripts';
135
136	// No SHOW TABLES guard: this is the chat hot path and the table is
137	// created on activation (with an admin-load safety net). A genuinely
138	// missing table fails the query and yields the same empty history the
139	// old option read produced on a fresh session.
140	$rows = $wpdb->get_results(
141	$wpdb->prepare(
142	"SELECT id, role, message, user_identifier, timestamp
143	FROM `$table` WHERE session_id = %s ORDER BY id ASC",
144	$session_id
145	),
146	ARRAY_A
147	);
148
149	$history = array();
150	if (is_array($rows)) {
151	foreach ($rows as $row) {
152	// The column stores GMT (current_time('mysql', 1) at the writer),
153	// so pin the parse to UTC rather than the site timezone.
154	$ts = strtotime($row['timestamp'] . ' +0000');
155	$history[] = array(
156	'id' => (int) $row['id'],
157	'role' => (string) $row['role'],
158	'content' => (string) $row['message'],
159	'timestamp' => ($ts ? $ts : 0) * 1000,
160	'agent_name' => (string) $row['user_identifier'],
161	);
162	}
163	}
164
165	self::$history_cache[$session_id] = $history;
166
167	return $history;
168	}
169
170	/**
171	* Drop the cached history for one session (or all). The writer calls this
172	* after every insert so a later read in the same request — e.g. the AI
173	* context build that follows saving the user's message — sees the new row,
174	* matching the read-your-own-write behavior update_option() gave the old
175	* option copy.
176	*
177	* @param string\|null $session_id Null flushes everything (test seam).
178	*/
179	public static function flush_session_history_cache($session_id = null) {
180	if ($session_id === null) {
181	self::$history_cache = array();
182	return;
183	}
184
185	unset(self::$history_cache[(string) $session_id]);
186	}
187
188	/**
189	* Append one message to a session's transcript on behalf of an add-on
190	* (plan 4a664e). The Forms add-on's wizard steps travel over the add-on's
191	* own AJAX action, never through the chat request, so nothing wrote them
192	* to the transcript: Transcripts, the live-agent handoff card, Download
193	* Transcript and every transcript email showed "[WIZARD_COMPLETE]" and no
194	* answers. This is the write half of get_session_history() — same row
195	* shape as the chat writer (visitor identity, captured name/email and
196	* originating page from the session store, GMT timestamp), without the
197	* new-session side effects (ownership, notification email, delayed
198	* transcript) that only the visitor's first message should trigger.
199	*
200	* Public and static so add-ons can call it, guarded with method_exists
201	* against an older mxchat-basic.
202	*
203	* @param string $session_id Chat session id.
204	* @param string $role user\|bot\|system\|agent.
205	* @param string $message Stored as-is: bot rows may carry HTML, user rows
206	* are rendered as text by every reader.
207	* @return int The new transcripts row id, 0 when nothing was written.
208	*/
209	public static function save_session_message($session_id, $role, $message) {
210	global $wpdb;
211
212	$session_id = self::sanitize_session_id($session_id);
213	$role = is_string($role) ? strtolower(trim($role)) : '';
214	$message = is_scalar($message) ? trim((string) $message) : '';
215
216	if ($session_id === '' \|\| $message === ''
217	\|\| !in_array($role, array('user', 'bot', 'system', 'agent'), true)) {
218	return 0;
219	}
220
221	$table = $wpdb->prefix . 'mxchat_chat_transcripts';
222
223	$saved_email = MxChat_Session_Store::get($session_id, 'email');
224	$saved_name = MxChat_Session_Store::get($session_id, 'name');
225
226	$row = array(
227	'user_id' => is_user_logged_in() ? get_current_user_id() : 0,
228	'user_identifier' => MxChat_User::mxchat_get_user_identifier(),
229	'user_email' => $saved_email ?: MxChat_User::mxchat_get_user_email(),
230	'user_name' => $saved_name ?: '',
231	'session_id' => $session_id,
232	'role' => $role,
233	'message' => $message,
234	'timestamp' => current_time('mysql', 1),
235	);
236
237	// Same page attribution as the rest of the session; the column pair was
238	// added by an upgrade routine, so tolerate its absence like the chat
239	// writer does.
240	$originating = MxChat_Session_Store::get($session_id, 'originating_page');
241	if (is_array($originating) && !empty($originating['url'])
242	&& $wpdb->get_var("SHOW COLUMNS FROM `$table` LIKE 'originating_page_url'")) {
243	$row['originating_page_url'] = $originating['url'];
244	$row['originating_page_title'] = isset($originating['title']) ? $originating['title'] : '';
245	}
246
247	$inserted = $wpdb->insert($table, $row);
248	if ($inserted === false) {
249	return 0;
250	}
251
252	$row_id = (int) $wpdb->insert_id;
253	self::flush_session_history_cache($session_id);
254
255	return $row_id;
256	}
257
258	/**
259	* Most recipients the Notification Email field will accept (plan 2f131a).
260	* A settings field is not a mailing list.
261	*/
262	const NOTIFICATION_EMAIL_MAX = 5;
263
264	/**
265	* Parse the Notification Email field into a list of recipients (plan 2f131a).
266	*
267	* THE TRAP THIS EXISTS TO CLOSE: sanitize_email() cannot be the validator for
268	* this field, because its output for the failing input is VALID. WordPress
269	* strips the separator and the surplus '@' and concatenates the remains:
270	*
271	* [email protected], [email protected] -> [email protected]
272	*
273	* and is_email() then returns true on that. So every guard in the plugin passed,
274	* the address was stored, the autosave ticked green, and both the new-session
275	* notification and the auto-emailed transcript went to a domain that does not
276	* exist — with no error anywhere. Validating the RAW part BEFORE sanitizing is
277	* the whole point; reversing those two lines silently restores the bug.
278	*
279	* All-or-nothing by design: if any entry is bad the caller must store NOTHING.
280	* A partial accept — keeping the good addresses and dropping the bad one — is
281	* the same defect in a new costume, because the owner still believes everyone
282	* on their list is being notified.
283	*
284	* @param mixed $raw Raw field value, exactly as submitted.
285	* @return array{emails: string[], error: string} Empty emails + empty error
286	* means the field was empty.
287	*/
288	public static function parse_notification_emails($raw) {
289	$out = array('emails' => array(), 'error' => '');
290
291	if (!is_scalar($raw)) {
292	$out['error'] = __('The notification email could not be read.', 'mxchat');
293	return $out;
294	}
295
296	$raw = trim((string) $raw);
297	if ($raw === '') {
298	return $out; // genuinely empty — the caller falls back to admin_email
299	}
300
301	$seen = array();
302	foreach (preg_split('/[,;]/', $raw) as $part) {
303	$part = trim($part);
304	if ($part === '') {
305	// A trailing or doubled separator carries no address, so skipping it
306	// cannot silently drop a recipient. This is the ONLY thing tolerated.
307	continue;
308	}
309
310	// RAW first. See the note above — order is load-bearing.
311	$clean = is_email($part) ? sanitize_email($part) : '';
312	if ($clean === '' \|\| !is_email($clean)) {
313	return array(
314	'emails' => array(),
315	'error' => sprintf(
316	/* translators: %s: the email address the owner typed. */
317	__('"%s" is not a valid email address, so nothing was saved. Separate multiple addresses with a comma.', 'mxchat'),
318	esc_html($part)
319	),
320	);
321	}
322
323	$key = strtolower($clean);
324	if (isset($seen[$key])) {
325	continue; // same address twice would simply mail them twice
326	}
327	$seen[$key] = true;
328	$out['emails'][] = $clean;
329	}
330
331	if (count($out['emails']) > self::NOTIFICATION_EMAIL_MAX) {
332	return array(
333	'emails' => array(),
334	'error' => sprintf(
335	/* translators: %d: maximum number of notification recipients. */
336	__('Enter at most %d email addresses, separated by commas.', 'mxchat'),
337	self::NOTIFICATION_EMAIL_MAX
338	),
339	);
340	}
341
342	return $out;
343	}
344
345	/**
346	* The stored recipient list, ready to hand to wp_mail() (plan 2f131a).
347	*
348	* Fallback rule, and it is narrow on purpose: an EMPTY field falls back to the
349	* site admin address, because that is the documented behaviour and an owner who
350	* never filled the field in still wants their notifications. A field holding
351	* something unusable does NOT fall back — it sends nowhere, exactly as before
352	* this plan. Falling back on bad input would mean a typo silently redirects a
353	* store's transcripts to a different mailbox than the one on screen.
354	*
355	* @param array\|null $options mxchat_transcripts_options, or null to read it.
356	* @return string[] Recipients; empty means do not send.
357	*/
358	public static function notification_recipients($options = null) {
359	if (!is_array($options)) {
360	$options = get_option('mxchat_transcripts_options', array());
361	if (!is_array($options)) {
362	$options = array();
363	}
364	}
365
366	$raw = isset($options['mxchat_notification_email']) ? $options['mxchat_notification_email'] : '';
367	$raw = is_scalar($raw) ? trim((string) $raw) : '';
368
369	if ($raw === '') {
370	$admin = get_option('admin_email');
371	return is_email($admin) ? array($admin) : array();
372	}
373
374	$parsed = self::parse_notification_emails($raw);
375	return $parsed['error'] === '' ? $parsed['emails'] : array();
376	}
377
378	/**
379	* Centralized embedding model registry. Single source of truth for dimensions
380	* and provider, so model-switch protection logic doesn't drift across files.
381	*/
382	public static function embedding_model_registry() {
383	return array(
384	'text-embedding-ada-002' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'Ada 2'),
385	'text-embedding-3-small' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'TE3 Small'),
386	'text-embedding-3-large' => array('dims' => 3072, 'provider' => 'openai', 'label' => 'TE3 Large'),
387	'voyage-3-large' => array('dims' => 2048, 'provider' => 'voyage', 'label' => 'Voyage-3 Large'),
388	'gemini-embedding-001' => array('dims' => 1536, 'provider' => 'gemini', 'label' => 'Gemini Embedding'),
389	);
390	}
391
392	public static function embedding_model_dimensions($model) {
393	$registry = self::embedding_model_registry();
394	return isset($registry[$model]) ? (int) $registry[$model]['dims'] : 0;
395	}
396
397	public static function embedding_model_label($model) {
398	if (is_string($model) && strpos($model, 'custom:') === 0) {
399	/* translators: %s: the embedding model name configured on the custom provider */
400	return sprintf(__('%s (custom provider)', 'mxchat'), substr($model, 7));
401	}
402	$registry = self::embedding_model_registry();
403	return isset($registry[$model]) ? $registry[$model]['label'] : $model;
404	}
405
406	/**
407	* Returns the model that was last used to actually write embeddings into the
408	* KB. Differs from the user-selected setting once a switch has happened but
409	* no re-embed has occurred yet — that's the mismatch state we warn about.
410	*/
411	public static function get_active_embedding_model() {
412	return get_option('mxchat_active_embedding_model', '');
413	}
414
415	/**
416	* Stamp the model that produced the most recent successful embedding. Called
417	* from generate_embedding() right after the API responds with a valid vector.
418	*/
419	public static function stamp_active_embedding_model($model) {
420	if (!empty($model) && $model !== self::get_active_embedding_model()) {
421	update_option('mxchat_active_embedding_model', $model, false);
422	}
423	}
424
425	/**
426	* The model name the custom-provider embedding path will send, mirroring the
427	* fallback chain the request itself uses: dedicated custom embedding model,
428	* else the custom chat model, else 'default'. Single source shared by
429	* generate_embedding_custom() and the mismatch-warning "selected" side so the
430	* two can never drift (plan ae02cb).
431	*/
432	public static function resolve_custom_embedding_model($options) {
433	if (isset($options['custom_provider_embedding_model']) && trim((string) $options['custom_provider_embedding_model']) !== '') {
434	return trim((string) $options['custom_provider_embedding_model']);
435	}
436	if (isset($options['custom_provider_model']) && trim((string) $options['custom_provider_model']) !== '') {
437	return trim((string) $options['custom_provider_model']);
438	}
439	return 'default';
440	}
441
442	/**
443	* The EFFECTIVE selected embedding model — what the next embed will actually
444	* use. With custom-provider embeddings on this is the custom identity in the
445	* same 'custom:<model>' form stamp_active_embedding_model() records, not the
446	* inert standard dropdown value. Mismatch-warning comparisons must read this,
447	* never $options['embedding_model'] directly — the dropdown cannot be
448	* deselected, so reading it raw flags every correctly-configured custom setup.
449	*/
450	public static function get_selected_embedding_model($options = null) {
451	if (!is_array($options)) {
452	$options = get_option('mxchat_options', array());
453	}
454	if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
455	return 'custom:' . self::resolve_custom_embedding_model($options);
456	}
457	return $options['embedding_model'] ?? '';
458	}
459
460	/**
461	* Extract the 11-character YouTube video ID from a URL, or '' if the URL is
462	* not a single-video YouTube link. Single source of truth for both the KB
463	* ingestion side and the chat render side — do not duplicate this parsing.
464	* Channel, playlist, and search URLs deliberately return '' (only a URL that
465	* identifies one video can be embedded).
466	*/
467	public static function parse_youtube_id($url) {
468	if (!is_string($url) \|\| $url === '' \|\| !preg_match('#^https?://#i', $url)) {
469	return '';
470	}
471	$host = strtolower((string) wp_parse_url($url, PHP_URL_HOST));
472	$host = preg_replace('/^(www\|m)\./', '', $host);
473	$path = (string) wp_parse_url($url, PHP_URL_PATH);
474	$id = '';
475	if ($host === 'youtu.be') {
476	$segments = explode('/', ltrim($path, '/'));
477	$id = $segments[0] ?? '';
478	} elseif (in_array($host, array('youtube.com', 'youtube-nocookie.com'), true)) {
479	if (preg_match('#^/(?:shorts\|embed\|live\|v)/([A-Za-z0-9_-]+)#', $path, $m)) {
480	$id = $m[1];
481	} elseif ($path === '/watch') {
482	parse_str((string) wp_parse_url($url, PHP_URL_QUERY), $query_vars);
483	$id = isset($query_vars['v']) ? (string) $query_vars['v'] : '';
484	}
485	}
486	$id = preg_replace('/[^A-Za-z0-9_-]/', '', (string) $id);
487	return (strlen($id) === 11) ? $id : '';
488	}
489
490	/**
491	* plan-mxchat-20260813-f52492 — video-card gating.
492	*
493	* Two standalone options (NOT mxchat_options — they skip the sanitize/autosave
494	* traps entirely), read here so the gate in the integrator and the fields on
495	* Knowledge -> Chunking & Retrieval can never disagree about a default.
496	*
497	* Master switch. Default ON: the card is existing behavior, and this is an
498	* opt-out for owners who never want one, not a new feature to opt into.
499	*/
500	public static function video_embed_enabled() {
501	return get_option('mxchat_video_embed_enabled', 'on') === 'on';
502	}
503
504	/**
505	* The video card's OWN confidence floor, as a 0-1 cosine — deliberately not
506	* the site-wide Similarity Threshold (default 35). "Good enough to quote in
507	* the answer" and "good enough to put a video on screen" are different
508	* questions: retrieval is allowed to be generous because the model still
509	* decides what to say, whereas the card is asserted to the visitor with no
510	* such filter. Stored as an int percentage to match the site-wide slider's
511	* convention; the default (55) sits above it on purpose.
512	*
513	* MXCHAT_VIDEO_EMBED_THRESHOLD_DEFAULT is the single source of that number.
514	*/
515	public static function video_embed_threshold() {
516	$stored = get_option('mxchat_video_embed_threshold', null);
517	$percent = ($stored === null \|\| $stored === '')
518	? MXCHAT_VIDEO_EMBED_THRESHOLD_DEFAULT
519	: (int) $stored;
520	if ($percent < 0) { $percent = 0; }
521	if ($percent > 100) { $percent = 100; }
522	// Cast: PHP evaluates 100/100 to int(1), so an unclamped return type would
523	// vary with the stored value. Callers compare against a cosine — keep it float.
524	return (float) $percent / 100;
525	}
526
527	/**
528	* UPDATED: Submit or update content (and its embedding) in the database.
529	* Stores in Pinecone if enabled, otherwise stores in WordPress DB.
530	*
531	* @param string $content The content to be embedded.
532	* @param string $source_url The source URL of the content.
533	* @param string $api_key The API key used for generating embeddings.
534	* @param string $vector_id Optional vector ID for Pinecone (if not provided, will use md5 of URL)
535	* @param string $bot_id The bot ID for multi-bot support
536	* @param string $content_type The type of content (post, page, pdf, url, manual, product, etc.)
537	* @return bool\|WP_Error True on success, WP_Error on failure
538	*/
539	public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
540	global $wpdb;
541	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
542
543	//error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ', Type: ' . $content_type . ')');
544	//error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
545
546	// Sanitize the source URL. Internal identities (mxchat:// manual docs,
547	// upload:// file uploads) are NOT web URLs — esc_url_raw EMPTIES them
548	// because its protocol list is statically cached and effectively
549	// unfilterable (measured, plan 945406 / 0485e5) — sanitize those as text
550	// so upserts stay keyed to a stable identity across re-imports.
551	if (preg_match('#^(mxchat\|upload)://#i', $source_url)) {
552	$source_url = sanitize_text_field($source_url);
553	} else {
554	$source_url = esc_url_raw($source_url);
555	}
556
557	// Sanitize content_type
558	$content_type = sanitize_key($content_type);
559	if (empty($content_type)) {
560	$content_type = 'content'; // Fallback for backwards compatibility
561	}
562
563	// Just ensure UTF-8 validity without aggressive escaping
564	$safe_content = wp_check_invalid_utf8($content);
565	// Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D)
566	$safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
567
568	// Check if chunking should be applied
569	$chunker = MxChat_Chunker::from_settings();
570	if ($chunker->should_chunk($safe_content)) {
571	//error_log('[MXCHAT-DB] Content exceeds chunk threshold, using chunked submission');
572	$chunk_result = self::submit_chunked_content($safe_content, $source_url, $api_key, $bot_id, $content_type, $chunker);
573	// Vector Store mirror gets the entry WHOLE — one file per KB entry,
574	// OpenAI chunks server-side; local chunking is our own embedding
575	// concern and never reaches the store (plan 15b5c6).
576	if ($chunk_result === true && class_exists('MxChat_Vectorstore_Manager')) {
577	MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type);
578	}
579	return $chunk_result;
580	}
581
582	// UPDATED: Generate the embedding using bot-specific configuration
583	$embedding_vector = self::generate_embedding($content, $api_key, $bot_id);
584
585	if (!is_array($embedding_vector)) {
586	// Surface the provider's real reason instead of a fixed string (4a7c0a).
587	$reason = is_wp_error($embedding_vector)
588	? $embedding_vector->get_error_message()
589	: 'Failed to generate embedding for content';
590	return new WP_Error('embedding_failed', $reason);
591	}
592
593	//error_log('[MXCHAT-DB] Embedding generated successfully');
594
595	// UPDATED: Check if Pinecone is enabled for this specific bot
596	if (self::is_pinecone_enabled_for_bot($bot_id)) {
597	//error_log('[MXCHAT-DB] Pinecone is enabled for bot ' . $bot_id . ' - using Pinecone storage');
598	// Store in Pinecone only
599	$result = self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id, $content_type);
600	// If this URL previously stored as CHUNKED content and the new content fits in a
601	// single vector, the upsert above only overwrote the base id — the old
602	// md5(url)_chunk_N vectors would keep serving the stale text. Sweep them.
603	// Only for a real source_url: chunk ids derive from it, and md5('') is shared
604	// by legacy URL-less entries so a blind sweep there could hit other entries.
605	if ($result === true && !empty($source_url)) {
606	self::cleanup_pinecone_chunk_stragglers($source_url, $bot_id);
607	}
608	if ($result === true && class_exists('MxChat_Vectorstore_Manager')) {
609	MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type);
610	}
611	return $result;
612	} else {
613	//error_log('[MXCHAT-DB] Pinecone not enabled for bot ' . $bot_id . ' - using WordPress storage');
614	// Store in WordPress database only
615	$embedding_vector_serialized = maybe_serialize($embedding_vector);
616	$result = self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type);
617	if ($result === true && class_exists('MxChat_Vectorstore_Manager')) {
618	MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type);
619	}
620	return $result;
621	}
622	}
623
624	/**
625	* UPDATED: Check if Pinecone is enabled and properly configured for a specific bot
626	*/
627	private static function is_pinecone_enabled_for_bot($bot_id = 'default') {
628	// For default bot or when multi-bot is not active, use original method
629	if ($bot_id === 'default' \|\| !class_exists('MxChat_Multi_Bot_Manager')) {
630	return self::is_pinecone_enabled();
631	}
632
633	// Get bot-specific Pinecone configuration
634	$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
635
636	if (empty($bot_pinecone_config)) {
637	// Fallback to default configuration
638	return self::is_pinecone_enabled();
639	}
640
641	$enabled_check = !empty($bot_pinecone_config['use_pinecone']) && $bot_pinecone_config['use_pinecone'];
642	$api_key_check = !empty($bot_pinecone_config['api_key']);
643	$host_check = !empty($bot_pinecone_config['host']);
644
645	return $enabled_check && $api_key_check && $host_check;
646	}
647
648	/**
649	* Check if Pinecone is enabled and properly configured (original method for default bot)
650	*/
651	private static function is_pinecone_enabled() {
652	$pinecone_options = get_option('mxchat_pinecone_addon_options');
653
654	if (empty($pinecone_options)) {
655	return false;
656	}
657
658	$enabled_check = !empty($pinecone_options['mxchat_use_pinecone']) && $pinecone_options['mxchat_use_pinecone'] !== '0';
659	$api_key_check = !empty($pinecone_options['mxchat_pinecone_api_key']);
660	$host_check = !empty($pinecone_options['mxchat_pinecone_host']);
661
662	return $enabled_check && $api_key_check && $host_check;
663	}
664
665	/**
666	* UPDATED: Store content in Pinecone only with bot support
667	*/
668	private static function store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
669	//error_log('[MXCHAT-PINECONE] ===== Using Pinecone-only storage for bot ' . $bot_id . ' =====');
670
671	// Get bot-specific Pinecone configuration
672	if ($bot_id === 'default' \|\| !class_exists('MxChat_Multi_Bot_Manager')) {
673	$pinecone_options = get_option('mxchat_pinecone_addon_options');
674	$api_key = $pinecone_options['mxchat_pinecone_api_key'];
675	$environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
676	$index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
677	$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
678	} else {
679	$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
680	if (empty($bot_pinecone_config)) {
681	// Fallback to default configuration
682	$pinecone_options = get_option('mxchat_pinecone_addon_options');
683	$api_key = $pinecone_options['mxchat_pinecone_api_key'];
684	$environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
685	$index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
686	$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
687	} else {
688	$api_key = $bot_pinecone_config['api_key'];
689	$environment = ''; // Not used in new Pinecone API
690	$index_name = ''; // Not used in new Pinecone API
691	$namespace = $bot_pinecone_config['namespace'] ?? '';
692	}
693	}
694
695	$result = self::store_in_pinecone_main(
696	$embedding_vector,
697	$content,
698	$source_url,
699	$api_key,
700	$environment,
701	$index_name,
702	$vector_id,
703	$bot_id,
704	$namespace,
705	$content_type
706	);
707
708	if (is_wp_error($result)) {
709	//error_log('[MXCHAT-PINECONE] Pinecone storage failed for bot ' . $bot_id . ': ' . $result->get_error_message());
710	return $result;
711	}
712
713	//error_log('[MXCHAT-PINECONE] Pinecone storage completed successfully for bot ' . $bot_id);
714	return true;
715	}
716
717	/**
718	* Store content in WordPress database with progressive fallback
719	* UPDATED 2.5.6: Now includes content_type parameter
720	*/
721	private static function store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type = 'content') {
722	global $wpdb;
723
724	//error_log('[MXCHAT-DB] ===== Using WordPress-only storage =====');
725
726	// Sanitize content_type
727	$content_type = sanitize_key($content_type);
728	if (empty($content_type)) {
729	$content_type = 'content'; // Fallback for backwards compatibility
730	}
731
732	// ===== FIXED: Generate unique identifier for manual content =====
733	$original_source_url = $source_url;
734	// Check if this is truly manual content (no URL at all) vs a real URL that filter_var rejects
735	// filter_var(FILTER_VALIDATE_URL) rejects valid URLs with encoded chars, non-ASCII, fragments, etc.
736	// Use a looser check: if it starts with http(s):// or has a scheme, it's a URL
737	$has_url_scheme = !empty($source_url) && preg_match('#^https?://#i', $source_url);
738	// upload:// identities (admin document/PDF uploads, plan 0485e5) are stable
739	// and deduplicable — treat them like URLs so a re-upload UPDATES the row
740	// instead of minting a fresh manual identity (which would duplicate).
741	$has_stable_identity = $has_url_scheme \|\| (!empty($source_url) && preg_match('#^upload://#i', $source_url));
742	// Treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
743	$is_legacy_mxchat_url = $has_url_scheme && strpos($source_url, 'mxchat.ai') !== false;
744	$is_manual_content = empty($source_url) \|\| $source_url === '' \|\| !$has_stable_identity \|\| $is_legacy_mxchat_url;
745
746	if ($is_manual_content) {
747	// Generate unique identifier for manual content to prevent overwrites
748	$source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
749	//error_log('[MXCHAT-DB] Generated unique ID for manual content: ' . $source_url);
750	}
751
752	// Only check for duplicates if we have a valid source URL (not manual content)
753	$existing_id = null;
754	if (!$is_manual_content) {
755	$existing_id = $wpdb->get_var(
756	$wpdb->prepare(
757	"SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
758	$source_url
759	)
760	);
761	//error_log('[MXCHAT-DB] Checked for existing URL, found ID: ' . ($existing_id ?: 'none'));
762	} else {
763	//error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)');
764	}
765	// ===== END FIX =====
766
767	// Progressive fallback mechanism for problematic content
768	$attempt = 1;
769	$max_attempts = 3;
770	$current_content = $safe_content;
771	$result = false;
772
773	while ($attempt <= $max_attempts && $result === false) {
774	try {
775	if ($existing_id) {
776	//error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
777
778	// Update the existing row - UPDATED 2.5.6: Added content_type
779	$result = $wpdb->update(
780	$table_name,
781	array(
782	'url' => $source_url,
783	'article_content' => $current_content,
784	'embedding_vector' => $embedding_vector_serialized,
785	'source_url' => $source_url,
786	'content_type' => $content_type,
787	'timestamp' => current_time('mysql'),
788	),
789	array('id' => $existing_id),
790	array('%s','%s','%s','%s','%s','%s'),
791	array('%d')
792	);
793	} else {
794	//error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
795	//error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
796
797	// Insert a new row - UPDATED 2.5.6: Added content_type
798	$result = $wpdb->insert(
799	$table_name,
800	array(
801	'url' => $source_url, // Now unique for manual content
802	'article_content' => $current_content,
803	'embedding_vector' => $embedding_vector_serialized,
804	'source_url' => $source_url, // Now unique for manual content
805	'content_type' => $content_type,
806	'timestamp' => current_time('mysql'),
807	),
808	array('%s','%s','%s','%s','%s','%s')
809	);
810	}
811
812	if ($result === false) {
813	//error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . ')');
814	//error_log('[MXCHAT-DB] MySQL Error: ' . $wpdb->last_error);
815	//error_log('[MXCHAT-DB] MySQL Error Number: ' . $wpdb->last_errno);
816	//error_log('[MXCHAT-DB] Last Query: ' . substr($wpdb->last_query, 0, 500));
817	//error_log('[MXCHAT-DB] Content length: ' . strlen($current_content) . ' bytes');
818	//error_log('[MXCHAT-DB] Embedding vector length: ' . strlen($embedding_vector_serialized) . ' bytes');
819
820	// Progressively apply more aggressive sanitization on failure
821	if ($attempt === 1) {
822	// First fallback: Use a more aggressive character filter and shorten
823	$current_content = preg_replace('/[^\p{L}\p{N}\s.,;:!?()-]/u', '', $current_content);
824	$current_content = substr($current_content, 0, 50000);
825	} else if ($attempt === 2) {
826	// Second fallback: Keep only alphanumeric and basic punctuation, shorten further
827	$current_content = preg_replace('/[^a-zA-Z0-9\s.,;:!?()-]/u', '', $current_content);
828	$current_content = substr($current_content, 0, 30000);
829	}
830
831	$attempt++;
832	}
833	} catch (Exception $e) {
834	//error_log('[MXCHAT-DB] Exception during database operation: ' . $e->getMessage());
835	$attempt++;
836	}
837	}
838
839	if ($result === false) {
840	//error_log('[MXCHAT-DB] All database operation attempts failed');
841	//error_log('[MXCHAT-DB] Final MySQL Error: ' . $wpdb->last_error);
842
843	$detailed_error = sprintf(
844	'Failed to store content in WordPress database after %d attempts. MySQL Error: %s (Error #%d). Content size: %d bytes, Embedding size: %d bytes',
845	$max_attempts,
846	$wpdb->last_error,
847	$wpdb->last_errno,
848	strlen($current_content),
849	strlen($embedding_vector_serialized)
850	);
851
852	return new WP_Error('database_failed', $detailed_error);
853	}
854
855	//error_log('[MXCHAT-DB] WordPress database operation completed successfully (Attempt ' . ($attempt - 1) . ')');
856	return true;
857	}
858
859	/**
860	* UPDATED: Store content in Pinecone database with bot support
861	* UPDATED 2.5.6: Now accepts content_type parameter
862	*/
863	private static function store_in_pinecone_main($embedding_vector, $content, $url, $api_key, $environment, $index_name, $vector_id = null, $bot_id = 'default', $namespace = '', $content_type = 'content') {
864	//error_log('[MXCHAT-PINECONE-MAIN] ===== Starting Pinecone storage for bot ' . $bot_id . ' =====');
865
866	// ===== UPDATED: Handle manual content with unique vector IDs =====
867	if ($vector_id) {
868	// Use provided vector ID
869	//error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
870	} elseif (!empty($url) && preg_match('#^(https?\|upload)://#i', $url)) {
871	// For URLs — and stable upload:// identities (plan 0485e5) — use an
872	// identity-derived ID so a re-import upserts the same vector.
873	$vector_id = md5($url);
874	//error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
875	} else {
876	// For manual content (empty/no URL scheme), generate unique ID
877	$vector_id = 'manual_' . time() . '_' . substr(md5($content . microtime(true)), 0, 8);
878	//error_log('[MXCHAT-PINECONE-MAIN] Generated unique vector ID for manual content: ' . $vector_id);
879	}
880	// ===== END UPDATE =====
881
882	// Get host from bot-specific config or fallback to default
883	if ($bot_id === 'default' \|\| !class_exists('MxChat_Multi_Bot_Manager')) {
884	$options = get_option('mxchat_pinecone_addon_options');
885	$host = $options['mxchat_pinecone_host'] ?? '';
886	} else {
887	$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
888	if (!empty($bot_pinecone_config)) {
889	$host = $bot_pinecone_config['host'] ?? '';
890	} else {
891	$options = get_option('mxchat_pinecone_addon_options');
892	$host = $options['mxchat_pinecone_host'] ?? '';
893	}
894	}
895
896	//error_log('[MXCHAT-PINECONE-MAIN] Host: ' . $host);
897	//error_log('[MXCHAT-PINECONE-MAIN] API key length: ' . strlen($api_key));
898	//error_log('[MXCHAT-PINECONE-MAIN] Bot ID: ' . $bot_id);
899	//error_log('[MXCHAT-PINECONE-MAIN] Namespace: ' . $namespace);
900
901	if (empty($host)) {
902	//error_log('[MXCHAT-PINECONE-MAIN] ERROR: Host is empty');
903	return new WP_Error('pinecone_config', 'Pinecone host is not configured. Please set the host in your bot settings.');
904	}
905
906	// ===== UPDATED 2.5.6: Use passed content_type or determine from URL if not provided =====
907	// Sanitize content_type
908	$content_type = sanitize_key($content_type);
909	if (empty($content_type)) {
910	// Fallback to old detection logic for backwards compatibility
911	$is_product = false;
912	$content_type = 'manual'; // Default for manual content
913
914	if (!empty($url) && preg_match('#^https?://#i', $url)) {
915	$is_product = (strpos($url, '/product/') !== false \|\| strpos($url, '/shop/') !== false);
916	$content_type = $is_product ? 'product' : 'content';
917	}
918	}
919
920	//error_log('[MXCHAT-PINECONE-MAIN] Content type: ' . $content_type);
921	// ===== END UPDATE =====
922
923	$api_endpoint = "https://{$host}/vectors/upsert";
924	//error_log('[MXCHAT-PINECONE-MAIN] API endpoint: ' . $api_endpoint);
925
926	// UPDATED 2.5.6: Use provided content_type in metadata
927	$metadata = array(
928	'text' => $content,
929	'source_url' => $url, // Can be empty for manual content
930	'type' => $content_type, // Now supports: post, page, pdf, url, manual, product, etc.
931	'last_updated' => time(),
932	'created_at' => time(), // Add creation timestamp
933	'bot_id' => $bot_id, // Add bot identification
934	);
935
936	// Document index (plan 362c31): same id, same metadata, through the
937	// documents API. A site on the classic vector index never enters here.
938	if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) {
939	return MxChat_Pinecone_Documents::upsert_one($vector_id, $content, $embedding_vector, $metadata, array(
940	'api_key' => $api_key,
941	'host' => $host,
942	'namespace' => $namespace,
943	));
944	}
945
946	$vector_data = array(
947	'id' => $vector_id,
948	'values' => $embedding_vector,
949	'metadata' => $metadata
950	);
951
952	$request_body = array(
953	'vectors' => array($vector_data)
954	);
955
956	// Add namespace if specified for multi-bot separation
957	if (!empty($namespace)) {
958	$request_body['namespace'] = $namespace;
959	//error_log('[MXCHAT-PINECONE-MAIN] Using namespace: ' . $namespace);
960	}
961
962	//error_log('[MXCHAT-PINECONE-MAIN] Request body prepared (embedding dimensions: ' . count($embedding_vector) . ')');
963
964	$response = wp_remote_post($api_endpoint, array(
965	'headers' => array(
966	'Api-Key' => $api_key,
967	'accept' => 'application/json',
968	'content-type' => 'application/json'
969	),
970	'body' => wp_json_encode($request_body),
971	'timeout' => 30,
972	'data_format' => 'body'
973	));
974
975	if (is_wp_error($response)) {
976	//error_log('[MXCHAT-PINECONE-MAIN] WordPress request error: ' . $response->get_error_message());
977	return new WP_Error('pinecone_request', $response->get_error_message());
978	}
979
980	$response_code = wp_remote_retrieve_response_code($response);
981	//error_log('[MXCHAT-PINECONE-MAIN] Response code: ' . $response_code);
982
983	if ($response_code !== 200) {
984	$body = wp_remote_retrieve_body($response);
985	//error_log('[MXCHAT-PINECONE-MAIN] API error - Response body: ' . $body);
986	return new WP_Error('pinecone_api', sprintf(
987	'Pinecone API error (HTTP %d): %s',
988	$response_code,
989	$body
990	));
991	}
992
993	$response_body = wp_remote_retrieve_body($response);
994	//error_log('[MXCHAT-PINECONE-MAIN] Success response: ' . $response_body);
995	//error_log('[MXCHAT-PINECONE-MAIN] Successfully stored in Pinecone for bot ' . $bot_id);
996	//error_log('[MXCHAT-PINECONE-MAIN] ===== Pinecone storage complete =====');
997
998	return true;
999	}
1000
1001	/**
1002	* Caller-side pre-flight for KB ingestion: can an embedding request be made
1003	* with these options, and which API key should travel downstream?
1004	*
1005	* Custom-provider-aware — generate_embedding() below routes to the custom
1006	* endpoint FIRST and ignores the passed cloud key entirely when
1007	* custom_provider_for_embeddings is on, so on that branch the only real
1008	* requirement is a Base URL. Ingestion callers that gated on a cloud API key
1009	* were killing keyless custom-embeddings sites (local Ollama / LM Studio
1010	* class) before the embed layer could route (plan cbd5fd).
1011	*
1012	* NOTE: reads $options['embedding_model'] raw on purpose — this mirrors
1013	* generate_embedding()'s own routing read, NOT the mismatch-banner's
1014	* "selected" chain (get_selected_embedding_model). The helper must predict
1015	* what the very next embed call will do, byte-for-byte.
1016	*
1017	* Decision only — callers keep their own error-surfacing shape (admin-notice
1018	* transient + redirect, wp_send_json_error, WP_Error, silent return).
1019	*
1020	* @param array\|null $options Resolved options (bot-specific where the caller
1021	* has them); null loads the default bot's options.
1022	* @return array {
1023	* @type bool $ok Whether ingestion can proceed.
1024	* @type string $api_key Key to pass downstream ('' on the custom branch —
1025	* generate_embedding() ignores it there).
1026	* @type string $reason Human-readable blocker; '' when $ok.
1027	* @type string $provider Short provider label ('OpenAI', 'Voyage AI',
1028	* 'Google Gemini', 'Custom Provider').
1029	* }
1030	*/
1031	public static function embedding_preflight($options = null) {
1032	if (!is_array($options)) {
1033	$options = get_option('mxchat_options');
1034	$options = is_array($options) ? $options : array();
1035	}
1036
1037	// Custom branch mirrors generate_embedding()'s routing order (custom first).
1038	if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
1039	$base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
1040	if ($base_url === '') {
1041	return array(
1042	'ok' => false,
1043	'api_key' => '',
1044	// Same string generate_embedding_custom() returns for this state.
1045	'reason' => __('Custom provider Base URL is not configured.', 'mxchat'),
1046	'provider' => 'Custom Provider',
1047	);
1048	}
1049	return array('ok' => true, 'api_key' => '', 'reason' => '', 'provider' => 'Custom Provider');
1050	}
1051
1052	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
1053	if (strpos($selected_model, 'voyage') === 0) {
1054	$api_key = $options['voyage_api_key'] ?? '';
1055	$provider = 'Voyage AI';
1056	} elseif (strpos($selected_model, 'gemini-embedding') === 0) {
1057	$api_key = $options['gemini_api_key'] ?? '';
1058	$provider = 'Google Gemini';
1059	} else {
1060	$api_key = $options['api_key'] ?? '';
1061	$provider = 'OpenAI';
1062	}
1063
1064	if (empty($api_key)) {
1065	return array(
1066	'ok' => false,
1067	'api_key' => '',
1068	'reason' => sprintf(
1069	/* translators: %s: embedding provider name */
1070	__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
1071	$provider
1072	),
1073	'provider' => $provider,
1074	);
1075	}
1076
1077	return array('ok' => true, 'api_key' => $api_key, 'reason' => '', 'provider' => $provider);
1078	}
1079
1080	/**
1081	* Public QUERY-side entry point (plan 876edb). The chat pipeline's
1082	* MxChat_Integrator::mxchat_generate_embedding() adapter routes through here
1083	* so the query and index sides share ONE provider-routing implementation —
1084	* the same endpoints, request bodies, and stamping semantics. The Integrator
1085	* keeps its own error vocabulary by translating the WP_Error this returns
1086	* (see the structured error data on every failure path below).
1087	*
1088	* @param string $text The text to be embedded.
1089	* @param string $api_key Caller-resolved API key (per-bot on the query side).
1090	* @param string $bot_id The bot ID for multi-bot support.
1091	* @return array\|WP_Error The embedding vector, or WP_Error carrying the reason.
1092	*/
1093	public static function generate_query_embedding($text, $api_key, $bot_id = 'default') {
1094	return self::generate_embedding($text, $api_key, $bot_id);
1095	}
1096
1097	/**
1098	* UPDATED: Generate an embedding for the given text using bot-specific configuration.
1099	*
1100	* @param string $text The text to be embedded.
1101	* @param string $api_key The API key used for generating embeddings.
1102	* @param string $bot_id The bot ID for multi-bot support
1103	* @return array\|null The embedding vector or null on failure.
1104	*/
1105	private static function generate_embedding($text, $api_key, $bot_id = 'default') {
1106	// Get bot-specific options
1107	if ($bot_id === 'default' \|\| !class_exists('MxChat_Multi_Bot_Manager')) {
1108	$options = get_option('mxchat_options');
1109	} else {
1110	$bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
1111	$options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
1112	}
1113
1114	// Opt-in: when the custom provider is selected for embeddings, route the KB
1115	// INDEX side through the same custom endpoint the query side uses, so stored
1116	// vectors and query vectors come from the same model. Default-off behavior
1117	// below is untouched.
1118	if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
1119	$custom = self::generate_embedding_custom($text, $options);
1120	// The custom path already returns a human-readable error string —
1121	// carry it instead of collapsing to null (plan 4a7c0a). The 'custom'
1122	// branch marker lets the Integrator adapter map the string back onto
1123	// its own error codes (876edb).
1124	return is_array($custom) ? $custom : new WP_Error('embedding_failed', (string) $custom, array('branch' => 'custom'));
1125	}
1126
1127	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
1128
1129	// Determine endpoint and API key based on model
1130	if (strpos($selected_model, 'voyage') === 0) {
1131	$endpoint = 'https://api.voyageai.com/v1/embeddings';
1132	$api_key = $options['voyage_api_key'] ?? '';
1133	} elseif (strpos($selected_model, 'gemini-embedding') === 0) {
1134	$endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
1135	$api_key = $options['gemini_api_key'] ?? '';
1136	} else {
1137	$endpoint = 'https://api.openai.com/v1/embeddings';
1138	// Prefer the caller-resolved key when one was passed — the query side
1139	// resolves per-bot keys at its call sites (integrator adapter, 876edb).
1140	// Index callers pass the preflight key, which equals this options read,
1141	// so nothing changes for them.
1142	$api_key = !empty($api_key) ? $api_key : ($options['api_key'] ?? '');
1143	}
1144
1145	// Prepare request body based on provider
1146	if (strpos($selected_model, 'gemini-embedding') === 0) {
1147	// Gemini API format
1148	$request_body = [
1149	'model' => 'models/' . $selected_model,
1150	'content' => [
1151	'parts' => [
1152	['text' => $text]
1153	]
1154	],
1155	'outputDimensionality' => 1536
1156	];
1157
1158	// Prepare headers for Gemini (API key as query parameter)
1159	$endpoint .= '?key=' . $api_key;
1160	$headers = [
1161	'Content-Type' => 'application/json'
1162	];
1163	} else {
1164	// OpenAI/Voyage API format
1165	$request_body = [
1166	'input' => $text,
1167	'model' => $selected_model
1168	];
1169
1170	// Add output_dimension for voyage-3-large
1171	if ($selected_model === 'voyage-3-large') {
1172	$request_body['output_dimension'] = 2048;
1173	}
1174
1175	// Prepare headers for OpenAI/Voyage
1176	$headers = [
1177	'Content-Type' => 'application/json',
1178	'Authorization' => 'Bearer ' . $api_key
1179	];
1180	}
1181
1182	$args = [
1183	'body' => wp_json_encode($request_body),
1184	'headers' => $headers,
1185	'timeout' => 60,
1186	'redirection' => 5,
1187	'blocking' => true,
1188	'httpversion' => '1.0',
1189	'sslverify' => true,
1190	];
1191
1192	$response = wp_remote_post($endpoint, $args);
1193
1194	if (is_wp_error($response)) {
1195	$message = 'Embedding request failed (connection): ' . $response->get_error_message();
1196	if (class_exists('MxChat_Admin')) {
1197	MxChat_Admin::mxchat_log_debug('embedding_error', $message, array('model' => $selected_model, 'bot_id' => $bot_id));
1198	}
1199	return new WP_Error('embedding_failed', $message, array(
1200	'branch' => 'cloud',
1201	'kind' => 'connection',
1202	'reason' => $response->get_error_message(),
1203	'model' => $selected_model,
1204	));
1205	}
1206
1207	$response_body = json_decode(wp_remote_retrieve_body($response), true);
1208
1209	// Handle different response formats based on provider
1210	if (strpos($selected_model, 'gemini-embedding') === 0) {
1211	// Gemini API response format
1212	if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
1213	self::stamp_active_embedding_model($selected_model);
1214	return $response_body['embedding']['values'];
1215	} else {
1216	return self::embedding_failure_error($response, $selected_model, $api_key, $bot_id);
1217	}
1218	} else {
1219	// OpenAI/Voyage API response format
1220	if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
1221	self::stamp_active_embedding_model($selected_model);
1222	return $response_body['data'][0]['embedding'];
1223	} else {
1224	return self::embedding_failure_error($response, $selected_model, $api_key, $bot_id);
1225	}
1226	}
1227	}
1228
1229	/**
1230	* Build a WP_Error carrying the embedding provider's REAL failure reason,
1231	* and record it in the Debug Mode log. Previously every failure path
1232	* returned bare null, so customers saw only "Failed to generate embedding
1233	* for content" / "Failed to store any chunks" with no cause (plan 4a7c0a).
1234	*
1235	* The API key never appears in provider response bodies (it travels in the
1236	* request headers), but the reason is scrubbed for it anyway before it can
1237	* reach a notice or the debug log.
1238	*/
1239	private static function embedding_failure_error($response, $selected_model, $api_key, $bot_id) {
1240	$status = (int) wp_remote_retrieve_response_code($response);
1241	$raw = (string) wp_remote_retrieve_body($response);
1242	$decoded = json_decode($raw, true);
1243
1244	// Provider error shapes: OpenAI + Gemini use {"error":{"message":…}};
1245	// Voyage uses {"detail":…}.
1246	$reason = '';
1247	if (is_array($decoded)) {
1248	if (isset($decoded['error']['message']) && is_string($decoded['error']['message'])) {
1249	$reason = $decoded['error']['message'];
1250	} elseif (isset($decoded['detail']) && is_string($decoded['detail'])) {
1251	$reason = $decoded['detail'];
1252	}
1253	}
1254	if ($reason === '') {
1255	$reason = ($raw !== '') ? substr($raw, 0, 200) : 'empty or malformed response';
1256	}
1257	if (is_string($api_key) && $api_key !== '') {
1258	$reason = str_replace($api_key, '[redacted]', $reason);
1259	}
1260	$reason = substr($reason, 0, 300);
1261	$message = sprintf('Embedding failed (%s, HTTP %d): %s', $selected_model, $status, $reason);
1262
1263	if (class_exists('MxChat_Admin')) {
1264	MxChat_Admin::mxchat_log_debug('embedding_error', $message, array(
1265	'model' => $selected_model,
1266	'status' => $status,
1267	'bot_id' => $bot_id,
1268	));
1269	}
1270
1271	// Structured data so the Integrator's query-side adapter can rebuild its
1272	// typed error contract (auth/rate-limit/quota/invalid-response) without a
1273	// second transport implementation (876edb). Additive — message unchanged.
1274	return new WP_Error('embedding_failed', $message, array(
1275	'branch' => 'cloud',
1276	'status' => $status,
1277	'error_type' => (is_array($decoded) && isset($decoded['error']['type']) && is_string($decoded['error']['type'])) ? $decoded['error']['type'] : '',
1278	'reason' => $reason,
1279	'model' => $selected_model,
1280	));
1281	}
1282
1283	/**
1284	* Generate an embedding via a Custom (OpenAI-compatible) provider's /embeddings route.
1285	* Shared by every embedding entry point so the KNOWLEDGE-BASE INDEX side and the
1286	* QUERY side route through the same model when the opt-in
1287	* 'custom_provider_for_embeddings' setting is on. Mirrors the query-path logic in
1288	* MxChat_Integrator::mxchat_generate_embedding_custom() but takes an explicit
1289	* $options array so it is callable statically from utils + knowledge-manager.
1290	*
1291	* Returns a numeric array (the embedding vector) on success, or a human-readable
1292	* error string on failure (so callers expecting a string error, like the
1293	* knowledge-manager, can surface it directly; callers expecting array\|null wrap it).
1294	*
1295	* @param string $text Text to embed.
1296	* @param array $options The resolved mxchat options (must contain the custom_provider_* keys).
1297	* @return array\|string Embedding vector on success; error string on failure.
1298	*/
1299	public static function generate_embedding_custom($text, $options) {
1300	if (empty($text)) {
1301	return 'No text provided for embedding generation';
1302	}
1303
1304	$base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
1305	if (empty($base_url)) {
1306	return 'Custom provider Base URL is not configured.';
1307	}
1308
1309	$api_key = isset($options['custom_provider_api_key']) ? trim((string) $options['custom_provider_api_key']) : '';
1310	$auth_scheme = isset($options['custom_provider_auth_scheme']) ? $options['custom_provider_auth_scheme'] : 'bearer';
1311	$api_version = isset($options['custom_provider_api_version']) ? trim((string) $options['custom_provider_api_version']) : '';
1312
1313	// Embedding model: shared resolver (dedicated embedding model -> chat model
1314	// -> 'default') — the mismatch warning's "selected" side reads the same chain.
1315	$model = self::resolve_custom_embedding_model($options);
1316
1317	$embed_url = $base_url . '/embeddings';
1318	if (!empty($api_version)) {
1319	$embed_url .= (strpos($embed_url, '?') === false ? '?' : '&') . 'api-version=' . rawurlencode($api_version);
1320	}
1321
1322	$headers = ['Content-Type' => 'application/json'];
1323	if (!empty($api_key)) {
1324	if ($auth_scheme === 'api-key') {
1325	$headers['api-key'] = $api_key;
1326	} else {
1327	$headers['Authorization'] = 'Bearer ' . $api_key;
1328	}
1329	}
1330
1331	$response = wp_remote_post($embed_url, [
1332	'headers' => $headers,
1333	'body' => wp_json_encode(['input' => $text, 'model' => $model]),
1334	'timeout' => 60,
1335	]);
1336	if (is_wp_error($response)) {
1337	return self::log_custom_embedding_failure(
1338	'Connection error when generating embeddings (custom provider): ' . $response->get_error_message(),
1339	$model,
1340	$api_key
1341	);
1342	}
1343
1344	$status = wp_remote_retrieve_response_code($response);
1345	$body = json_decode(wp_remote_retrieve_body($response), true);
1346	if ($status !== 200) {
1347	$msg = isset($body['error']['message']) ? $body['error']['message'] : 'HTTP ' . $status;
1348	return self::log_custom_embedding_failure(
1349	'Custom embedding endpoint error: ' . $msg,
1350	$model,
1351	$api_key,
1352	(int) $status
1353	);
1354	}
1355	if (isset($body['data'][0]['embedding']) && is_array($body['data'][0]['embedding'])) {
1356	// Stamp the custom model identity so the active-embedding-model mismatch
1357	// warning reflects the real (custom) model rather than the built-in setting.
1358	self::stamp_active_embedding_model('custom:' . $model);
1359	return $body['data'][0]['embedding'];
1360	}
1361	return self::log_custom_embedding_failure('Invalid embedding response from custom provider.', $model, $api_key);
1362	}
1363
1364	/**
1365	* Record a custom-provider embedding failure in the Debug Mode log, then
1366	* return the message unchanged so callers keep their string-error contract.
1367	* The cloud branch has logged its failures since 4a7c0a; the custom branch
1368	* never did, so chat-side failures on Custom-provider installs were
1369	* invisible to Debug Mode despite the 3.2.18 readme saying otherwise
1370	* (plan 71e4b6). Same scrub-then-log shape as embedding_failure_error().
1371	*
1372	* @param string $message Human-readable failure (the caller's return value).
1373	* @param string $model Resolved custom embedding model.
1374	* @param string $api_key Scrubbed out of the logged message if it ever appears.
1375	* @param int $status HTTP status when one was received, 0 otherwise.
1376	* @return string The (scrubbed) message.
1377	*/
1378	private static function log_custom_embedding_failure($message, $model, $api_key, $status = 0) {
1379	if (is_string($api_key) && $api_key !== '') {
1380	$message = str_replace($api_key, '[redacted]', $message);
1381	}
1382
1383	if (class_exists('MxChat_Admin')) {
1384	$context = array('model' => 'custom:' . $model);
1385	if ($status > 0) {
1386	$context['status'] = $status;
1387	}
1388	MxChat_Admin::mxchat_log_debug('embedding_error', $message, $context);
1389	}
1390
1391	return $message;
1392	}
1393
1394	/**
1395	* Submit content as multiple chunks
1396	*
1397	* Splits large content into chunks, generates embeddings for each,
1398	* and stores them with chunk metadata for later reassembly.
1399	*
1400	* @param string $content The content to chunk and store
1401	* @param string $source_url The source URL
1402	* @param string $api_key The API key for embeddings
1403	* @param string $bot_id The bot ID
1404	* @param string $content_type The content type
1405	* @param MxChat_Chunker $chunker The chunker instance
1406	* @return bool\|WP_Error True on success, WP_Error on failure
1407	*/
1408	private static function submit_chunked_content($content, $source_url, $api_key, $bot_id, $content_type, $chunker) {
1409	global $wpdb;
1410	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1411
1412	//error_log('[MXCHAT-CHUNK-DEBUG] Starting chunked submission for: ' . $source_url);
1413	//error_log('[MXCHAT-CHUNK-DEBUG] Content length: ' . strlen($content) . ' chars');
1414
1415	// URL-less (manual) content needs a minted identity BEFORE chunk ids are derived:
1416	// every chunk id is md5(source_url)_chunk_N, so with source_url = '' EVERY long
1417	// manual document shared the md5('') prefix — and the clean-slate delete below
1418	// wiped the PREVIOUS manual entry's chunks each time a new one was added. The
1419	// single-vector paths already mint (mxchat:// in WP, manual_* in Pinecone); this
1420	// was the one storage path that didn't. Keep an mxchat:// identity if the caller
1421	// already carries one.
1422	if (strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, 'upload://') !== 0 && !preg_match('#^https?://#i', $source_url)) {
1423	$source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
1424	}
1425
1426	// First, delete any existing chunks for this URL (clean slate).
1427	// Mirror-suspended: this is a re-store, not an entry removal — the Vector
1428	// Store file is replaced (or kept, on hash match) by the caller's
1429	// sync_upsert_entry after storage succeeds.
1430	self::$vectorstore_mirror_suspended = true;
1431	$delete_result = self::delete_chunks_for_url($source_url, $bot_id);
1432	self::$vectorstore_mirror_suspended = false;
1433	if (is_wp_error($delete_result)) {
1434	//error_log('[MXCHAT-CHUNK-DEBUG] Warning: Failed to delete existing chunks: ' . $delete_result->get_error_message());
1435	// Continue anyway - we'll overwrite with upsert
1436	}
1437
1438	// Split content into chunks
1439	$chunks = $chunker->chunk_text($content);
1440	$total_chunks = count($chunks);
1441
1442	//error_log('[MXCHAT-CHUNK-DEBUG] Created ' . $total_chunks . ' chunks');
1443	foreach ($chunks as $i => $chunk) {
1444	//error_log('[MXCHAT-CHUNK-DEBUG] Chunk ' . $i . ' length: ' . strlen($chunk) . ' chars, preview: ' . substr($chunk, 0, 100));
1445	}
1446
1447	//error_log('[MXCHAT-CHUNK] Split content into ' . $total_chunks . ' chunks');
1448
1449	if ($total_chunks === 0) {
1450	return new WP_Error('chunking_failed', 'Content could not be split into chunks');
1451	}
1452
1453	$errors = array();
1454	$embed_failures = 0;
1455	$first_embed_reason = '';
1456	$first_store_reason = '';
1457	$is_pinecone = self::is_pinecone_enabled_for_bot($bot_id);
1458
1459	foreach ($chunks as $index => $chunk_text) {
1460	// Generate chunk metadata
1461	$chunk_metadata = MxChat_Chunker::create_chunk_metadata($index, $total_chunks, $source_url);
1462
1463	// AI-Engine-style aliases so external consumers (Pinecone/Qdrant/Chroma) can rely on
1464	// a stable shorthand ('source'/'part_index'/'part_total') without parsing our internal names.
1465	$chunk_metadata['source'] = $source_url;
1466	$chunk_metadata['part_index'] = (int) $index;
1467	$chunk_metadata['part_total'] = (int) $total_chunks;
1468
1469	/**
1470	* Filter the per-chunk metadata blob before it's written to the KB store.
1471	*
1472	* @param array $chunk_metadata Metadata array (source, part_index, part_total, chunk_index, total_chunks, source_url, parent_url_hash, document_type, ...).
1473	* @param string $chunk_text The chunk text being stored.
1474	* @param array $context ['bot_id' => string, 'content_type' => string, 'source_url' => string, 'part_index' => int, 'part_total' => int]
1475	* @return array Updated metadata array.
1476	*/
1477	$chunk_metadata = apply_filters(
1478	'mxchat_embedding_chunk_metadata',
1479	$chunk_metadata,
1480	$chunk_text,
1481	array(
1482	'bot_id' => $bot_id,
1483	'content_type' => $content_type,
1484	'source_url' => $source_url,
1485	'part_index' => (int) $index,
1486	'part_total' => (int) $total_chunks,
1487	)
1488	);
1489
1490	$chunk_vector_id = MxChat_Chunker::generate_chunk_vector_id($source_url, $index);
1491
1492	//error_log('[MXCHAT-CHUNK] Processing chunk ' . ($index + 1) . '/' . $total_chunks . ' (ID: ' . $chunk_vector_id . ')');
1493
1494	// Generate embedding for this chunk
1495	$embedding_vector = self::generate_embedding($chunk_text, $api_key, $bot_id);
1496
1497	if (!is_array($embedding_vector)) {
1498	// Track embedding failures separately from storage failures, and
1499	// keep the first provider reason seen — the two failure classes
1500	// have opposite remedies (API key vs Pinecone/DB) (plan 4a7c0a).
1501	$embed_failures++;
1502	$reason = is_wp_error($embedding_vector) ? $embedding_vector->get_error_message() : '';
1503	if ($reason !== '' && $first_embed_reason === '') {
1504	$first_embed_reason = $reason;
1505	}
1506	$errors[] = new WP_Error('embedding_failed', 'Failed to generate embedding for chunk ' . $index . ($reason !== '' ? ' — ' . $reason : ''));
1507	continue;
1508	}
1509
1510	if ($is_pinecone) {
1511	// Store in Pinecone with chunk metadata
1512	$result = self::store_chunk_in_pinecone(
1513	$embedding_vector,
1514	$chunk_text,
1515	$source_url,
1516	$chunk_vector_id,
1517	$bot_id,
1518	$content_type,
1519	$chunk_metadata
1520	);
1521	} else {
1522	// Store in WordPress DB with chunk metadata
1523	$content_with_metadata = MxChat_Chunker::format_chunk_for_storage($chunk_text, $chunk_metadata);
1524	$embedding_vector_serialized = maybe_serialize($embedding_vector);
1525
1526	$result = self::store_chunk_in_wordpress_db(
1527	$content_with_metadata,
1528	$source_url,
1529	$embedding_vector_serialized,
1530	$table_name,
1531	$content_type,
1532	$chunk_metadata
1533	);
1534	}
1535
1536	if (is_wp_error($result)) {
1537	$errors[] = $result;
1538	if ($first_store_reason === '') {
1539	$first_store_reason = $result->get_error_message();
1540	}
1541	}
1542	}
1543
1544	if (count($errors) === $total_chunks) {
1545	// Say WHICH stage failed — "failed to store" used to cover pure
1546	// embedding failures too, sending customers to debug Pinecone when
1547	// the problem was their embedding API key (plan 4a7c0a).
1548	if ($embed_failures === $total_chunks) {
1549	return new WP_Error('chunking_failed',
1550	'Failed to store any chunks — every chunk failed to embed'
1551	. ($first_embed_reason !== '' ? ': ' . $first_embed_reason : '')
1552	. ' Check the embedding provider API key and model under MxChat Settings.');
1553	}
1554	if ($embed_failures === 0) {
1555	return new WP_Error('chunking_failed',
1556	'Failed to store any chunks — embeddings generated but storage failed'
1557	. ($first_store_reason !== '' ? ': ' . $first_store_reason : '')
1558	. ' Check the knowledge base storage (Pinecone index or database).');
1559	}
1560	return new WP_Error('chunking_failed', sprintf(
1561	'Failed to store any chunks — %d failed to embed%s and %d failed to store%s',
1562	$embed_failures,
1563	$first_embed_reason !== '' ? ' (' . $first_embed_reason . ')' : '',
1564	$total_chunks - $embed_failures,
1565	$first_store_reason !== '' ? ' (' . $first_store_reason . ')' : ''
1566	));
1567	}
1568
1569	if (!empty($errors)) {
1570	$detail = $first_embed_reason !== '' ? $first_embed_reason : $first_store_reason;
1571	return new WP_Error('chunking_partial_failure',
1572	sprintf('Failed to store %d of %d chunks', count($errors), $total_chunks)
1573	. ($detail !== '' ? ' — first error: ' . $detail : ''));
1574	}
1575
1576	//error_log('[MXCHAT-CHUNK] Successfully stored all ' . $total_chunks . ' chunks');
1577	return true;
1578	}
1579
1580	/**
1581	* Store a single chunk in Pinecone with chunk-specific metadata
1582	*/
1583	private static function store_chunk_in_pinecone($embedding_vector, $chunk_text, $source_url, $vector_id, $bot_id, $content_type, $chunk_metadata) {
1584	// Get Pinecone configuration
1585	if ($bot_id === 'default' \|\| !class_exists('MxChat_Multi_Bot_Manager')) {
1586	$pinecone_options = get_option('mxchat_pinecone_addon_options');
1587	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1588	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1589	$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1590	} else {
1591	$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1592	if (empty($bot_pinecone_config)) {
1593	$pinecone_options = get_option('mxchat_pinecone_addon_options');
1594	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1595	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1596	$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1597	} else {
1598	$api_key = $bot_pinecone_config['api_key'] ?? '';
1599	$host = $bot_pinecone_config['host'] ?? '';
1600	$namespace = $bot_pinecone_config['namespace'] ?? '';
1601	}
1602	}
1603
1604	if (empty($host) \|\| empty($api_key)) {
1605	return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
1606	}
1607
1608	$api_endpoint = "https://{$host}/vectors/upsert";
1609
1610	// Build metadata with chunk information
1611	$metadata = array(
1612	'text' => $chunk_text,
1613	'source_url' => $source_url,
1614	'type' => $content_type,
1615	'is_chunked' => true,
1616	'chunk_index' => $chunk_metadata['chunk_index'],
1617	'total_chunks' => $chunk_metadata['total_chunks'],
1618	'parent_url_hash' => $chunk_metadata['parent_url_hash'],
1619	'last_updated' => time(),
1620	'created_at' => time(),
1621	'bot_id' => $bot_id,
1622	);
1623
1624	// Document index (plan 362c31): same chunk id and metadata, documents API.
1625	if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) {
1626	return MxChat_Pinecone_Documents::upsert_one($vector_id, $chunk_text, $embedding_vector, $metadata, array(
1627	'api_key' => $api_key,
1628	'host' => $host,
1629	'namespace' => $namespace,
1630	));
1631	}
1632
1633	$vector_data = array(
1634	'id' => $vector_id,
1635	'values' => $embedding_vector,
1636	'metadata' => $metadata
1637	);
1638
1639	$request_body = array(
1640	'vectors' => array($vector_data)
1641	);
1642
1643	if (!empty($namespace)) {
1644	$request_body['namespace'] = $namespace;
1645	}
1646
1647	$response = wp_remote_post($api_endpoint, array(
1648	'headers' => array(
1649	'Api-Key' => $api_key,
1650	'accept' => 'application/json',
1651	'content-type' => 'application/json'
1652	),
1653	'body' => wp_json_encode($request_body),
1654	'timeout' => 30
1655	));
1656
1657	if (is_wp_error($response)) {
1658	return $response;
1659	}
1660
1661	$response_code = wp_remote_retrieve_response_code($response);
1662	if ($response_code !== 200) {
1663	return new WP_Error('pinecone_api', 'Pinecone API error: HTTP ' . $response_code);
1664	}
1665
1666	return true;
1667	}
1668
1669	/**
1670	* Store a single chunk in WordPress database
1671	*/
1672	private static function store_chunk_in_wordpress_db($content_with_metadata, $source_url, $embedding_vector_serialized, $table_name, $content_type, $chunk_metadata) {
1673	global $wpdb;
1674
1675	// For chunks, we always insert new rows (no duplicate checking)
1676	// The URL includes chunk info in the metadata, but source_url stays the same for grouping
1677	$result = $wpdb->insert(
1678	$table_name,
1679	array(
1680	'url' => $source_url,
1681	'article_content' => $content_with_metadata,
1682	'embedding_vector' => $embedding_vector_serialized,
1683	'source_url' => $source_url,
1684	'content_type' => $content_type,
1685	'timestamp' => current_time('mysql')
1686	),
1687	array('%s', '%s', '%s', '%s', '%s', '%s')
1688	);
1689
1690	if ($result === false) {
1691	return new WP_Error('database_failed', 'Failed to insert chunk: ' . $wpdb->last_error);
1692	}
1693
1694	return true;
1695	}
1696
1697	/**
1698	* Delete all chunks for a given URL
1699	*
1700	* @param string $source_url The source URL
1701	* @param string $bot_id The bot ID
1702	* @return bool\|WP_Error True on success, WP_Error on failure
1703	*/
1704	public static function delete_chunks_for_url($source_url, $bot_id = 'default') {
1705	//error_log('[MXCHAT-CHUNK-DELETE] Deleting chunks for URL: ' . $source_url);
1706
1707	// Entry removal — mirror it to the Vector Store (unless a storage routine
1708	// is mid-re-store, see $vectorstore_mirror_suspended).
1709	if (!self::$vectorstore_mirror_suspended && class_exists('MxChat_Vectorstore_Manager')) {
1710	MxChat_Vectorstore_Manager::sync_delete_entry($source_url, $bot_id);
1711	}
1712
1713	if (self::is_pinecone_enabled_for_bot($bot_id)) {
1714	return self::delete_pinecone_chunks_by_url($source_url, $bot_id);
1715	} else {
1716	return self::delete_wordpress_chunks_by_url($source_url);
1717	}
1718	}
1719
1720	/**
1721	* Delete all chunks for a URL from Pinecone
1722	*/
1723	/**
1724	* Delete leftover md5(url)_chunk_N vectors after a URL's content was re-stored
1725	* as a SINGLE vector (content shrank below the chunk threshold on edit/re-import).
1726	* Unlike delete_pinecone_chunks_by_url this leaves the base id alone — the caller
1727	* just upserted the new content there. Failure is logged, not fatal: the save
1728	* itself succeeded, and the next save retries the sweep.
1729	*/
1730	private static function cleanup_pinecone_chunk_stragglers($source_url, $bot_id) {
1731	if ($bot_id === 'default' \|\| !class_exists('MxChat_Multi_Bot_Manager')) {
1732	$pinecone_options = get_option('mxchat_pinecone_addon_options');
1733	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1734	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1735	$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1736	} else {
1737	$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1738	if (empty($bot_pinecone_config)) {
1739	$pinecone_options = get_option('mxchat_pinecone_addon_options');
1740	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1741	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1742	$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1743	} else {
1744	$api_key = $bot_pinecone_config['api_key'] ?? '';
1745	$host = $bot_pinecone_config['host'] ?? '';
1746	$namespace = $bot_pinecone_config['namespace'] ?? '';
1747	}
1748	}
1749
1750	if (empty($host) \|\| empty($api_key)) {
1751	return;
1752	}
1753
1754	// Document index (plan 362c31): list the chunk ids and delete them through the documents API.
1755	if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) {
1756	$docs_cfg = array('api_key' => $api_key, 'host' => $host, 'namespace' => $namespace);
1757	$doc_stragglers = MxChat_Pinecone_Documents::list_ids(md5($source_url) . '_chunk_', $docs_cfg);
1758	if (!empty($doc_stragglers)) {
1759	$doc_result = MxChat_Pinecone_Documents::delete_ids($doc_stragglers, $docs_cfg);
1760	if (is_wp_error($doc_result) && class_exists('MxChat_Admin') && method_exists('MxChat_Admin', 'mxchat_log_debug')) {
1761	MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to sweep stale chunk documents after single-vector re-store', array('source_url' => $source_url, 'bot_id' => $bot_id, 'count' => count($doc_stragglers)));
1762	}
1763	}
1764	return;
1765	}
1766
1767	$stragglers = array();
1768
1769	// Pinecone /vectors/list is a GET endpoint with query-string parameters (a POST
1770	// answers 200-with-an-empty-body, which reads as "no stragglers").
1771	$query_params = array(
1772	'prefix' => md5($source_url) . '_chunk_',
1773	'limit' => 100,
1774	);
1775	if (!empty($namespace)) {
1776	$query_params['namespace'] = $namespace;
1777	}
1778
1779	$list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
1780
1781	do {
1782	$list_response = wp_remote_get($list_url, array(
1783	'headers' => array(
1784	'Api-Key' => $api_key,
1785	'accept' => 'application/json',
1786	),
1787	'timeout' => 30,
1788	));
1789
1790	if (is_wp_error($list_response) \|\| wp_remote_retrieve_response_code($list_response) !== 200) {
1791	break;
1792	}
1793
1794	$list_data = json_decode(wp_remote_retrieve_body($list_response), true);
1795	if (!empty($list_data['vectors'])) {
1796	foreach ($list_data['vectors'] as $vector) {
1797	if (isset($vector['id'])) {
1798	$stragglers[] = $vector['id'];
1799	}
1800	}
1801	}
1802
1803	$next_token = $list_data['pagination']['next'] ?? '';
1804	if (empty($next_token)) {
1805	break;
1806	}
1807
1808	$query_params['paginationToken'] = $next_token;
1809	$list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
1810	} while (true);
1811
1812	if (empty($stragglers)) {
1813	return;
1814	}
1815
1816	$delete_body = array('ids' => $stragglers);
1817	if (!empty($namespace)) {
1818	$delete_body['namespace'] = $namespace;
1819	}
1820
1821	$delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
1822	'headers' => array(
1823	'Api-Key' => $api_key,
1824	'accept' => 'application/json',
1825	'content-type' => 'application/json'
1826	),
1827	'body' => wp_json_encode($delete_body),
1828	'timeout' => 30
1829	));
1830
1831	if ((is_wp_error($delete_response) \|\| wp_remote_retrieve_response_code($delete_response) !== 200)
1832	&& class_exists('MxChat_Admin') && method_exists('MxChat_Admin', 'mxchat_log_debug')) {
1833	MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to sweep stale chunk vectors after single-vector re-store', array('source_url' => $source_url, 'bot_id' => $bot_id, 'count' => count($stragglers)));
1834	}
1835	}
1836
1837	/**
1838	* Resolve the Pinecone credentials for a bot: the bot's own config when the
1839	* multi-bot add-on supplies one, else the site-wide options. Extracted so the
1840	* delete path and the count path can never resolve different credentials.
1841	*
1842	* @param string $bot_id
1843	* @return array{api_key:string,host:string,namespace:string}
1844	*/
1845	private static function pinecone_config_for_bot($bot_id) {
1846	if ($bot_id === 'default' \|\| !class_exists('MxChat_Multi_Bot_Manager')) {
1847	$bot_pinecone_config = array();
1848	} else {
1849	$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1850	}
1851
1852	if (empty($bot_pinecone_config)) {
1853	$pinecone_options = get_option('mxchat_pinecone_addon_options');
1854	return array(
1855	'api_key' => $pinecone_options['mxchat_pinecone_api_key'] ?? '',
1856	'host' => $pinecone_options['mxchat_pinecone_host'] ?? '',
1857	'namespace' => $pinecone_options['mxchat_pinecone_namespace'] ?? '',
1858	// 'vector' \| 'document' (plan 362c31) — the delete / count paths branch on it.
1859	'index_type' => (($pinecone_options['mxchat_pinecone_index_type'] ?? 'vector') === 'document') ? 'document' : 'vector',
1860	);
1861	}
1862
1863	return array(
1864	'api_key' => $bot_pinecone_config['api_key'] ?? '',
1865	'host' => $bot_pinecone_config['host'] ?? '',
1866	'namespace' => $bot_pinecone_config['namespace'] ?? '',
1867	'index_type' => (($bot_pinecone_config['index_type'] ?? 'vector') === 'document') ? 'document' : 'vector',
1868	);
1869	}
1870
1871	/**
1872	* List every Pinecone vector id under a prefix, following pagination.
1873	*
1874	* Pinecone /vectors/list is a GET endpoint with query-string parameters; a POST here returns a
1875	* non-200 silently and we end up seeing only the base vector, leaving chunks orphaned.
1876	*
1877	* @param string $prefix Vector-id prefix to match.
1878	* @param array $cfg From pinecone_config_for_bot().
1879	* @return array List of vector ids (empty when none match or the API is unreachable).
1880	*/
1881	private static function pinecone_list_ids_by_prefix($prefix, array $cfg) {
1882	// Document index (plan 362c31): the documents API lists by prefix too.
1883	if (($cfg['index_type'] ?? 'vector') === 'document' && class_exists('MxChat_Pinecone_Documents')) {
1884	return MxChat_Pinecone_Documents::list_ids($prefix, $cfg);
1885	}
1886
1887	$ids = array();
1888
1889	$query_params = array(
1890	'prefix' => $prefix,
1891	'limit' => 100,
1892	);
1893	if (!empty($cfg['namespace'])) {
1894	$query_params['namespace'] = $cfg['namespace'];
1895	}
1896
1897	$list_url = "https://{$cfg['host']}/vectors/list?" . http_build_query($query_params);
1898
1899	// Paginate in case a URL has more than 100 chunks.
1900	do {
1901	$list_response = wp_remote_get($list_url, array(
1902	'headers' => array(
1903	'Api-Key' => $cfg['api_key'],
1904	'accept' => 'application/json',
1905	),
1906	'timeout' => 30,
1907	));
1908
1909	if (is_wp_error($list_response) \|\| wp_remote_retrieve_response_code($list_response) !== 200) {
1910	break;
1911	}
1912
1913	$list_data = json_decode(wp_remote_retrieve_body($list_response), true);
1914	if (!empty($list_data['vectors'])) {
1915	foreach ($list_data['vectors'] as $vector) {
1916	if (isset($vector['id'])) {
1917	$ids[] = $vector['id'];
1918	}
1919	}
1920	}
1921
1922	$next_token = $list_data['pagination']['next'] ?? '';
1923	if (empty($next_token)) {
1924	break;
1925	}
1926
1927	$query_params['paginationToken'] = $next_token;
1928	$list_url = "https://{$cfg['host']}/vectors/list?" . http_build_query($query_params);
1929	} while (true);
1930
1931	return $ids;
1932	}
1933
1934	/**
1935	* How many stored pieces a URL currently has, in whichever backend holds it.
1936	*
1937	* Branches on the SAME condition delete_chunks_for_url() branches on, so the
1938	* number reported is the number that call removes. Used by the REST DELETE
1939	* route to answer "was anything actually there?" without a second delete path.
1940	*
1941	* @param string $source_url
1942	* @param string $bot_id
1943	* @return int
1944	*/
1945	public static function count_chunks_for_url($source_url, $bot_id = 'default') {
1946	if (self::is_pinecone_enabled_for_bot($bot_id)) {
1947	$cfg = self::pinecone_config_for_bot($bot_id);
1948	if (empty($cfg['host']) \|\| empty($cfg['api_key'])) {
1949	return 0;
1950	}
1951	// The base id and every "<base>_chunk_N" share the base as a prefix,
1952	// so one prefix listing covers chunked and non-chunked content alike.
1953	return count(self::pinecone_list_ids_by_prefix(md5($source_url), $cfg));
1954	}
1955
1956	global $wpdb;
1957	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1958
1959	return (int) $wpdb->get_var($wpdb->prepare(
1960	"SELECT COUNT(*) FROM {$table_name} WHERE source_url = %s",
1961	$source_url
1962	));
1963	}
1964
1965	private static function delete_pinecone_chunks_by_url($source_url, $bot_id) {
1966	$cfg = self::pinecone_config_for_bot($bot_id);
1967	$api_key = $cfg['api_key'];
1968	$host = $cfg['host'];
1969	$namespace = $cfg['namespace'];
1970
1971	if (empty($host) \|\| empty($api_key)) {
1972	return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
1973	}
1974
1975	// Document index (plan 362c31): base id + every <md5>_chunk_N, documents API.
1976	if (($cfg['index_type'] ?? 'vector') === 'document' && class_exists('MxChat_Pinecone_Documents')) {
1977	$doc_ids = array_merge(array(md5($source_url)), MxChat_Pinecone_Documents::list_ids(md5($source_url) . '_chunk_', $cfg));
1978	return MxChat_Pinecone_Documents::delete_ids($doc_ids, $cfg);
1979	}
1980
1981	$base_vector_id = md5($source_url);
1982	$vectors_to_delete = array();
1983
1984	// Add the original single-vector ID (for non-chunked content)
1985	$vectors_to_delete[] = $base_vector_id;
1986
1987	$vectors_to_delete = array_merge(
1988	$vectors_to_delete,
1989	self::pinecone_list_ids_by_prefix($base_vector_id . '_chunk_', $cfg)
1990	);
1991
1992	if (empty($vectors_to_delete)) {
1993	//error_log('[MXCHAT-CHUNK-DELETE] No vectors found to delete');
1994	return true;
1995	}
1996
1997	//error_log('[MXCHAT-CHUNK-DELETE] Deleting ' . count($vectors_to_delete) . ' vectors from Pinecone');
1998
1999	// Delete vectors
2000	$delete_url = "https://{$host}/vectors/delete";
2001
2002	$delete_body = array(
2003	'ids' => $vectors_to_delete
2004	);
2005
2006	if (!empty($namespace)) {
2007	$delete_body['namespace'] = $namespace;
2008	}
2009
2010	$delete_response = wp_remote_post($delete_url, array(
2011	'headers' => array(
2012	'Api-Key' => $api_key,
2013	'accept' => 'application/json',
2014	'content-type' => 'application/json'
2015	),
2016	'body' => wp_json_encode($delete_body),
2017	'timeout' => 30
2018	));
2019
2020	if (is_wp_error($delete_response)) {
2021	return $delete_response;
2022	}
2023
2024	$response_code = wp_remote_retrieve_response_code($delete_response);
2025	if ($response_code !== 200) {
2026	return new WP_Error('pinecone_delete', 'Failed to delete vectors: HTTP ' . $response_code);
2027	}
2028
2029	return true;
2030	}
2031
2032	/**
2033	* Delete all chunks for a URL from WordPress database
2034	*/
2035	private static function delete_wordpress_chunks_by_url($source_url) {
2036	global $wpdb;
2037	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2038
2039	// Delete all rows with this source_url (handles both chunked and non-chunked)
2040	$result = $wpdb->delete(
2041	$table_name,
2042	array('source_url' => $source_url),
2043	array('%s')
2044	);
2045
2046	if ($result === false) {
2047	return new WP_Error('database_delete', 'Failed to delete chunks: ' . $wpdb->last_error);
2048	}
2049
2050	//error_log('[MXCHAT-CHUNK-DELETE] Deleted ' . $result . ' rows from WordPress DB');
2051	return true;
2052	}
2053
2054	/**
2055	* Hybrid keyword boost (plan-38ffa1): detect whether the WP-DB knowledge
2056	* table can serve the keyword leg via a MySQL FULLTEXT index, creating the
2057	* index if needed. Detection runs once and caches the answer in the
2058	* mxchat_hybrid_keyword_capability option ('fulltext' \| 'like'); pass
2059	* $force to re-detect. LIKE is the graceful fallback for shared hosts
2060	* whose ALTER fails — the feature works either way, FULLTEXT just ranks
2061	* better and scales.
2062	*
2063	* @param bool $force Re-run detection even if a cached answer exists.
2064	* @return string 'fulltext' or 'like'
2065	*/
2066	public static function mxchat_hybrid_detect_capability($force = false) {
2067	$cached = get_option('mxchat_hybrid_keyword_capability', '');
2068	if (!$force && in_array($cached, array('fulltext', 'like'), true)) {
2069	return $cached;
2070	}
2071
2072	global $wpdb;
2073	$table = $wpdb->prefix . 'mxchat_system_prompt_content';
2074
2075	$index_exists = $wpdb->get_var("SHOW INDEX FROM {$table} WHERE Key_name = 'mxchat_content_ft'");
2076	if (!$index_exists) {
2077	// Suppress the visible error on hosts where this is not permitted —
2078	// failure is an expected, handled outcome (LIKE fallback).
2079	$suppress = $wpdb->suppress_errors(true);
2080	$wpdb->query("ALTER TABLE {$table} ADD FULLTEXT INDEX mxchat_content_ft (article_content)");
2081	$wpdb->suppress_errors($suppress);
2082	$index_exists = $wpdb->get_var("SHOW INDEX FROM {$table} WHERE Key_name = 'mxchat_content_ft'");
2083	}
2084
2085	$capability = $index_exists ? 'fulltext' : 'like';
2086	update_option('mxchat_hybrid_keyword_capability', $capability);
2087	return $capability;
2088	}
2089
2090	/**
2091	* Public entry for re-embedding already-stored content in place (wp mxchat
2092	* rtl-repair, plan d1e6f7). Thin wrapper so the repair CLI gets the exact
2093	* provider routing the import path uses — the repaired vector must come from
2094	* the same model family the bot indexes with, or retrieval stays broken.
2095	*/
2096	public static function regenerate_embedding($text, $api_key, $bot_id = 'default') {
2097	return self::generate_embedding($text, $api_key, $bot_id);
2098	}
2099
2100	/**
2101	* Restore logical character order in PDF-extracted RTL text (plan 32bf9e).
2102	*
2103	* The bundled Smalot parser only un-reverses text runs tagged with the
2104	* ReversedChars marked-content operator (Word emits it; LibreOffice and most
2105	* other producers do not), so their Hebrew/Arabic PDFs extract in visual
2106	* (reversed) order and embed/search as garbage. This is OUR post-processing
2107	* seam over getText() — the parser itself is never patched (it gets replaced
2108	* wholesale on library updates).
2109	*
2110	* Heuristic and deliberately conservative, per line:
2111	* - lines without strong RTL codepoints are untouched (a fully-Latin line in
2112	* an RTL document therefore stays as extracted — accepted limitation);
2113	* - Arabic presentation forms are a definitive visual-order signal (they only
2114	* appear in shaped output): de-shape to base letters and reverse;
2115	* - otherwise flip only on positive evidence — Hebrew final-letter position
2116	* (a sofit at word START only happens in reversed text) or sentence
2117	* punctuation position (leading in visual order, trailing in logical);
2118	* - ambiguous lines are left alone: a conservative miss beats corrupting a
2119	* Word-produced extraction the parser already handled (the double-flip
2120	* guard this plan's approval named mandatory).
2121	*
2122	* @param string $text One extracted page string, straight from getText().
2123	* @param string $context Caller tag for the Debug Mode entry (site + page).
2124	* @return string Text with RTL lines restored to logical order.
2125	*/
2126	public static function normalize_pdf_rtl($text, $context = '') {
2127	if (!is_string($text) \|\| '' === $text) {
2128	return $text;
2129	}
2130	// Escape hatch for sites whose PDFs already extract logically.
2131	if (!apply_filters('mxchat_pdf_rtl_normalize', true, $text)) {
2132	return $text;
2133	}
2134	// Fast bail: nothing RTL anywhere in the page.
2135	if (!preg_match('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $text)) {
2136	return $text;
2137	}
2138
2139	$parts = preg_split('/(\R)/u', $text, -1, PREG_SPLIT_DELIM_CAPTURE);
2140	if (false === $parts) {
2141	return $text;
2142	}
2143
2144	$flipped_lines = 0;
2145	$deshaped_lines = 0;
2146	foreach ($parts as $i => $part) {
2147	if ('' === $part \|\| preg_match('/^\R$/u', $part)) {
2148	continue;
2149	}
2150	$was_flipped = false;
2151	$was_deshaped = false;
2152	$new = self::pdf_rtl_normalize_line($part, $was_flipped, $was_deshaped);
2153	if ($new !== $part) {
2154	$parts[$i] = $new;
2155	}
2156	if ($was_flipped) {
2157	$flipped_lines++;
2158	}
2159	if ($was_deshaped) {
2160	$deshaped_lines++;
2161	}
2162	}
2163
2164	if (($flipped_lines \|\| $deshaped_lines) && class_exists('MxChat_Admin')) {
2165	MxChat_Admin::mxchat_log_debug('pdf_rtl_normalized', 'RTL PDF text restored to logical order', array(
2166	'context' => (string) $context,
2167	'lines_flipped' => $flipped_lines,
2168	'lines_deshaped' => $deshaped_lines,
2169	'decision' => 'visual-order extraction detected',
2170	));
2171	}
2172
2173	return implode('', $parts);
2174	}
2175
2176	/**
2177	* Normalize one line. Sets $flipped/$deshaped for the caller's debug entry.
2178	*/
2179	private static function pdf_rtl_normalize_line($line, &$flipped, &$deshaped) {
2180	$flipped = false;
2181	$deshaped = false;
2182
2183	if (!preg_match('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $line)) {
2184	return $line;
2185	}
2186
2187	$has_forms = (bool) preg_match('/[\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $line);
2188	$work = $line;
2189	if ($has_forms) {
2190	$work = strtr($work, self::pdf_rtl_deshape_map());
2191	$deshaped = ($work !== $line);
2192	}
2193
2194	$verdict = 'ambiguous';
2195	if ($has_forms) {
2196	// Shaped glyph codepoints only exist in visual-order output.
2197	$verdict = 'visual';
2198	} else {
2199	// Strong-direction dominance gate first: an LTR-dominant line with an
2200	// embedded RTL word is not flip material.
2201	$rtl_count = preg_match_all('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}]/u', $work, $m_rtl);
2202	$ltr_count = preg_match_all('/[A-Za-z]/u', $work, $m_ltr);
2203	if ($rtl_count < 1 \|\| $rtl_count <= $ltr_count) {
2204	return $line;
2205	}
2206
2207	// Hebrew final letters (ך ם ן ף ץ) end words in logical text; one at
2208	// a word START (Hebrew letter follows, none precedes) is reversal
2209	// evidence. Positional, so it survives the line being reversed.
2210	$sofit_initial = preg_match_all('/(?<![\x{05D0}-\x{05EA}])[\x{05DA}\x{05DD}\x{05DF}\x{05E3}\x{05E5}](?=[\x{05D0}-\x{05EA}])/u', $work, $m_i);
2211	$sofit_terminal = preg_match_all('/(?<=[\x{05D0}-\x{05EA}])[\x{05DA}\x{05DD}\x{05DF}\x{05E3}\x{05E5}](?![\x{05D0}-\x{05EA}])/u', $work, $m_t);
2212	if ($sofit_initial > $sofit_terminal) {
2213	$verdict = 'visual';
2214	} elseif ($sofit_terminal > $sofit_initial) {
2215	$verdict = 'logical';
2216	} else {
2217	// Sentence punctuation lands at the visual LEFT edge of an RTL
2218	// line, i.e. the START of a visual-order extraction.
2219	$trimmed = trim($work);
2220	$starts_punct = (bool) preg_match('/^[.?!:;,]/u', $trimmed);
2221	$ends_punct = (bool) preg_match('/[.?!:;,]$/u', $trimmed);
2222	if ($starts_punct && !$ends_punct) {
2223	$verdict = 'visual';
2224	} elseif ($ends_punct && !$starts_punct) {
2225	$verdict = 'logical';
2226	}
2227	}
2228	}
2229
2230	if ('visual' !== $verdict) {
2231	// Ambiguous or logical: hand back the original line UNLESS we
2232	// de-shaped (de-shaping alone is always safe — same letters, same
2233	// order, un-ligated).
2234	return $deshaped ? $work : $line;
2235	}
2236
2237	$flipped = true;
2238	return self::pdf_rtl_flip_line($work);
2239	}
2240
2241	/**
2242	* Reverse a visual-order line into logical order: full character reversal,
2243	* mirror paired punctuation, then re-reverse embedded LTR runs (Latin words
2244	* and digit sequences, incl. Arabic-Indic digits) so they stay readable.
2245	*/
2246	private static function pdf_rtl_flip_line($line) {
2247	$chars = preg_split('//u', $line, -1, PREG_SPLIT_NO_EMPTY);
2248	if (false === $chars) {
2249	return $line;
2250	}
2251	$reversed = implode('', array_reverse($chars));
2252	$reversed = strtr($reversed, array(
2253	'(' => ')', ')' => '(',
2254	'[' => ']', ']' => '[',
2255	'{' => '}', '}' => '{',
2256	'<' => '>', '>' => '<',
2257	));
2258	$restored = preg_replace_callback(
2259	'/[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9}](?:[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9} .,\'"%\-:\/]*[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9}])?/u',
2260	function ($m) {
2261	$run = preg_split('//u', $m[0], -1, PREG_SPLIT_NO_EMPTY);
2262	return false === $run ? $m[0] : implode('', array_reverse($run));
2263	},
2264	$reversed
2265	);
2266	return null === $restored ? $reversed : $restored;
2267	}
2268
2269	/**
2270	* Arabic presentation forms (A + B) -> base letters. Built once from range
2271	* specs rather than ~120 hand-written literal entries; every codepoint in a
2272	* range maps to the same base sequence (isolated/final/initial/medial forms
2273	* of one letter are contiguous in the FE70 block).
2274	*/
2275	private static function pdf_rtl_deshape_map() {
2276	static $map = null;
2277	if (null !== $map) {
2278	return $map;
2279	}
2280	$ranges = array(
2281	// Form B harakat (each pair = standalone + tatweel-joined form).
2282	array(0xFE70, 0xFE71, array(0x064B)), array(0xFE72, 0xFE72, array(0x064C)),
2283	array(0xFE74, 0xFE74, array(0x064D)), array(0xFE76, 0xFE77, array(0x064E)),
2284	array(0xFE78, 0xFE79, array(0x064F)), array(0xFE7A, 0xFE7B, array(0x0650)),
2285	array(0xFE7C, 0xFE7D, array(0x0651)), array(0xFE7E, 0xFE7F, array(0x0652)),
2286	// Form B letters.
2287	array(0xFE80, 0xFE80, array(0x0621)), array(0xFE81, 0xFE82, array(0x0622)),
2288	array(0xFE83, 0xFE84, array(0x0623)), array(0xFE85, 0xFE86, array(0x0624)),
2289	array(0xFE87, 0xFE88, array(0x0625)), array(0xFE89, 0xFE8C, array(0x0626)),
2290	array(0xFE8D, 0xFE8E, array(0x0627)), array(0xFE8F, 0xFE92, array(0x0628)),
2291	array(0xFE93, 0xFE94, array(0x0629)), array(0xFE95, 0xFE98, array(0x062A)),
2292	array(0xFE99, 0xFE9C, array(0x062B)), array(0xFE9D, 0xFEA0, array(0x062C)),
2293	array(0xFEA1, 0xFEA4, array(0x062D)), array(0xFEA5, 0xFEA8, array(0x062E)),
2294	array(0xFEA9, 0xFEAA, array(0x062F)), array(0xFEAB, 0xFEAC, array(0x0630)),
2295	array(0xFEAD, 0xFEAE, array(0x0631)), array(0xFEAF, 0xFEB0, array(0x0632)),
2296	array(0xFEB1, 0xFEB4, array(0x0633)), array(0xFEB5, 0xFEB8, array(0x0634)),
2297	array(0xFEB9, 0xFEBC, array(0x0635)), array(0xFEBD, 0xFEC0, array(0x0636)),
2298	array(0xFEC1, 0xFEC4, array(0x0637)), array(0xFEC5, 0xFEC8, array(0x0638)),
2299	array(0xFEC9, 0xFECC, array(0x0639)), array(0xFECD, 0xFED0, array(0x063A)),
2300	array(0xFED1, 0xFED4, array(0x0641)), array(0xFED5, 0xFED8, array(0x0642)),
2301	array(0xFED9, 0xFEDC, array(0x0643)), array(0xFEDD, 0xFEE0, array(0x0644)),
2302	array(0xFEE1, 0xFEE4, array(0x0645)), array(0xFEE5, 0xFEE8, array(0x0646)),
2303	array(0xFEE9, 0xFEEC, array(0x0647)), array(0xFEED, 0xFEEE, array(0x0648)),
2304	array(0xFEEF, 0xFEF0, array(0x0649)), array(0xFEF1, 0xFEF4, array(0x064A)),
2305	// Form B lam-alef ligatures decompose to two letters.
2306	array(0xFEF5, 0xFEF6, array(0x0644, 0x0622)), array(0xFEF7, 0xFEF8, array(0x0644, 0x0623)),
2307	array(0xFEF9, 0xFEFA, array(0x0644, 0x0625)), array(0xFEFB, 0xFEFC, array(0x0644, 0x0627)),
2308	// Form A: Persian / Urdu letters in common use.
2309	array(0xFB56, 0xFB59, array(0x067E)), array(0xFB66, 0xFB69, array(0x0679)),
2310	array(0xFB7A, 0xFB7D, array(0x0686)), array(0xFB88, 0xFB89, array(0x0688)),
2311	array(0xFB8A, 0xFB8B, array(0x0698)), array(0xFB8E, 0xFB91, array(0x06A9)),
2312	array(0xFB92, 0xFB95, array(0x06AF)), array(0xFBA6, 0xFBA9, array(0x06C1)),
2313	array(0xFBAA, 0xFBAD, array(0x06BE)), array(0xFBAE, 0xFBAF, array(0x06D2)),
2314	array(0xFBFC, 0xFBFF, array(0x06CC)),
2315	);
2316	$map = array();
2317	foreach ($ranges as $range) {
2318	$base = '';
2319	foreach ($range[2] as $cp) {
2320	$base .= self::pdf_rtl_cp_to_utf8($cp);
2321	}
2322	for ($cp = $range[0]; $cp <= $range[1]; $cp++) {
2323	$map[self::pdf_rtl_cp_to_utf8($cp)] = $base;
2324	}
2325	}
2326	return $map;
2327	}
2328
2329	/**
2330	* Codepoint to UTF-8 without ext-intl / mbstring entity tricks (PHP 7.2 floor).
2331	*/
2332	private static function pdf_rtl_cp_to_utf8($cp) {
2333	if ($cp < 0x80) {
2334	return chr($cp);
2335	}
2336	if ($cp < 0x800) {
2337	return chr(0xC0 \| ($cp >> 6)) . chr(0x80 \| ($cp & 0x3F));
2338	}
2339	if ($cp < 0x10000) {
2340	return chr(0xE0 \| ($cp >> 12)) . chr(0x80 \| (($cp >> 6) & 0x3F)) . chr(0x80 \| ($cp & 0x3F));
2341	}
2342	return chr(0xF0 \| ($cp >> 18)) . chr(0x80 \| (($cp >> 12) & 0x3F)) . chr(0x80 \| (($cp >> 6) & 0x3F)) . chr(0x80 \| ($cp & 0x3F));
2343	}
2344	}