mxchat-basic / includes / class-mxchat-utils.php

class-mxchat-utils.php in MxChat – AI Chatbot & Content Generation for WordPress 3.2.24, at includes/class-mxchat-utils.php

2,849 lines 119.7 KB

Raw Download Zip

1	<?php
2	if (!defined('ABSPATH')) {
3	exit; // Exit if accessed directly
4	}
5
6	class MxChat_Utils {
7
8	/**
9	* True while a storage routine is re-writing an entry it is about to re-add
10	* (submit_chunked_content's clean-slate delete). delete_chunks_for_url() skips
11	* the Vector Store mirror-delete while set — an internal re-store is not an
12	* entry removal, and mirroring it would delete-then-reupload the entry's file
13	* on every chunked save (plan 15b5c6).
14	*/
15	private static $vectorstore_mirror_suspended = false;
16
17	/**
18	* Validate a client-supplied session id (plan-mxchat-20260731-d42bec).
19	*
20	* sanitize_text_field() — which every session_id read site used before this —
21	* preserves '/' and '..'. Harmless where the value is only an option or
22	* transient key suffix, but mxchat_send_delayed_transcript() interpolates it
23	* into a filesystem path, so '../../../../path/x' wrote, emailed and deleted a
24	* file outside the uploads dir.
25	*
26	* REJECTS rather than rewrites: a silently-stripped id would orphan the
27	* conversation it belongs to, which is harder to diagnose than a clean refusal.
28	* Returns '' for anything malformed, so call sites fall into the empty-session
29	* error paths they already have.
30	*
31	* The generator only ever emits 'mxchat_chat_' + 32 hex chars
32	* (class-mxchat-integrator.php, js/chat-script.js), so this is not restrictive
33	* in practice. Length ceiling is deliberate — session ids are also used as
34	* option-name suffixes, and WP option names cap at 191 chars.
35	*
36	* @param mixed $raw Raw request value.
37	* @return string The id if well-formed, '' otherwise.
38	*/
39	public static function sanitize_session_id($raw) {
40	if (!is_scalar($raw)) {
41	return '';
42	}
43	$val = trim((string) $raw);
44	if ($val === '') {
45	return '';
46	}
47	return preg_match('/\A[A-Za-z0-9_-]{1,128}\z/', $val) ? $val : '';
48	}
49
50	/**
51	* Sanitize the lead-capture consent-checkbox label (plan b062c4).
52	*
53	* The label is owner-supplied and renders inside the widget's email form, so
54	* this is a security boundary, not a formatting nicety. One explicit
55	* allowlist, used at BOTH save time (options.php sanitize + autosave AJAX)
56	* and render time (widget form, admin surfaces) so the two can never drift:
57	* an anchor — the whole point is "I agree to the <a>Privacy Policy</a>" —
58	* plus inline emphasis. No block tags, no images, no style attributes.
59	*
60	* The stored consent record keeps this exact sanitized string as "the text
61	* the visitor saw", so it must be deterministic: same input, same output,
62	* whichever path ran it.
63	*
64	* @param mixed $raw Owner-entered label.
65	* @return string Sanitized label, capped at 1000 chars.
66	*/
67	public static function sanitize_consent_label($raw) {
68	if (!is_scalar($raw)) {
69	return '';
70	}
71
72	$allowed = array(
73	'a' => array(
74	'href' => true,
75	'title' => true,
76	'target' => true,
77	'rel' => true,
78	),
79	'strong' => array(),
80	'em' => array(),
81	'br' => array(),
82	);
83
84	$label = wp_kses(trim((string) $raw), $allowed);
85
86	return mb_substr($label, 0, 1000);
87	}
88
89	/**
90	* Per-request cache for get_session_history(). Mirrors get_option()'s
91	* request-scoped caching, which the mxchat_history_ option reads got for
92	* free before plan 839c4c moved history reads onto the transcripts table.
93	*/
94	private static $history_cache = array();
95
96	/**
97	* Session chat history read from the transcripts table, in the exact array
98	* shape the legacy mxchat_history_<sid> option stored (plan 839c4c). The
99	* option was a second copy of state the table already held — measured
100	* byte-identical in role/content/order on 174 of 177 real sessions, with
101	* the table a superset on the rest — at up to 64 KB per option row. The
102	* table is now the single store; nothing writes the option any more.
103	*
104	* Shape notes, load-bearing for the consumers:
105	* - id: the transcripts row id (int). Integer ids make the pollers'
106	* ">" comparisons correct where the old uniqid() strings only worked by
107	* accident of hex ordering.
108	* - timestamp: milliseconds, derived from the table's second-resolution GMT
109	* column (x1000). Consumers comparing against a real-millisecond client
110	* cutoff MUST floor the cutoff to the second and err inclusive — see the
111	* persistence-off filters in class-mxchat-integrator.php.
112	* - agent_name: the row's user_identifier, which the writer sets to the
113	* same displayed_name value the option carried (agent name when present,
114	* else email, else identifier).
115	*
116	* Public and static so mxchat-woo / mxchat-forms can call the same accessor
117	* as core, guarded with method_exists against an older mxchat-basic.
118	*
119	* @param string $session_id
120	* @return array[] Chronological entries: id, role, content, timestamp, agent_name.
121	*/
122	public static function get_session_history($session_id) {
123	global $wpdb;
124
125	$session_id = self::sanitize_session_id($session_id);
126	if ($session_id === '') {
127	return array();
128	}
129
130	if (array_key_exists($session_id, self::$history_cache)) {
131	return self::$history_cache[$session_id];
132	}
133
134	$table = $wpdb->prefix . 'mxchat_chat_transcripts';
135
136	// No SHOW TABLES guard: this is the chat hot path and the table is
137	// created on activation (with an admin-load safety net). A genuinely
138	// missing table fails the query and yields the same empty history the
139	// old option read produced on a fresh session.
140	$rows = $wpdb->get_results(
141	$wpdb->prepare(
142	"SELECT id, role, message, user_identifier, timestamp
143	FROM `$table` WHERE session_id = %s ORDER BY id ASC",
144	$session_id
145	),
146	ARRAY_A
147	);
148
149	$history = array();
150	if (is_array($rows)) {
151	foreach ($rows as $row) {
152	// The column stores GMT (current_time('mysql', 1) at the writer),
153	// so pin the parse to UTC rather than the site timezone.
154	$ts = strtotime($row['timestamp'] . ' +0000');
155	$history[] = array(
156	'id' => (int) $row['id'],
157	'role' => (string) $row['role'],
158	'content' => (string) $row['message'],
159	'timestamp' => ($ts ? $ts : 0) * 1000,
160	'agent_name' => (string) $row['user_identifier'],
161	);
162	}
163	}
164
165	self::$history_cache[$session_id] = $history;
166
167	return $history;
168	}
169
170	/**
171	* Drop the cached history for one session (or all). The writer calls this
172	* after every insert so a later read in the same request — e.g. the AI
173	* context build that follows saving the user's message — sees the new row,
174	* matching the read-your-own-write behavior update_option() gave the old
175	* option copy.
176	*
177	* @param string\|null $session_id Null flushes everything (test seam).
178	*/
179	public static function flush_session_history_cache($session_id = null) {
180	if ($session_id === null) {
181	self::$history_cache = array();
182	return;
183	}
184
185	unset(self::$history_cache[(string) $session_id]);
186	}
187
188	/**
189	* Append one message to a session's transcript on behalf of an add-on
190	* (plan 4a664e). The Forms add-on's wizard steps travel over the add-on's
191	* own AJAX action, never through the chat request, so nothing wrote them
192	* to the transcript: Transcripts, the live-agent handoff card, Download
193	* Transcript and every transcript email showed "[WIZARD_COMPLETE]" and no
194	* answers. This is the write half of get_session_history() — same row
195	* shape as the chat writer (visitor identity, captured name/email and
196	* originating page from the session store, GMT timestamp), without the
197	* new-session side effects (ownership, notification email, delayed
198	* transcript) that only the visitor's first message should trigger.
199	*
200	* Public and static so add-ons can call it, guarded with method_exists
201	* against an older mxchat-basic.
202	*
203	* @param string $session_id Chat session id.
204	* @param string $role user\|bot\|system\|agent.
205	* @param string $message Stored as-is: bot rows may carry HTML, user rows
206	* are rendered as text by every reader.
207	* @return int The new transcripts row id, 0 when nothing was written.
208	*/
209	public static function save_session_message($session_id, $role, $message) {
210	global $wpdb;
211
212	$session_id = self::sanitize_session_id($session_id);
213	$role = is_string($role) ? strtolower(trim($role)) : '';
214	$message = is_scalar($message) ? trim((string) $message) : '';
215
216	if ($session_id === '' \|\| $message === ''
217	\|\| !in_array($role, array('user', 'bot', 'system', 'agent'), true)) {
218	return 0;
219	}
220
221	$table = $wpdb->prefix . 'mxchat_chat_transcripts';
222
223	$saved_email = MxChat_Session_Store::get($session_id, 'email');
224	$saved_name = MxChat_Session_Store::get($session_id, 'name');
225
226	$row = array(
227	'user_id' => is_user_logged_in() ? get_current_user_id() : 0,
228	'user_identifier' => MxChat_User::mxchat_get_user_identifier(),
229	'user_email' => $saved_email ?: MxChat_User::mxchat_get_user_email(),
230	'user_name' => $saved_name ?: '',
231	'session_id' => $session_id,
232	'role' => $role,
233	'message' => $message,
234	'timestamp' => current_time('mysql', 1),
235	);
236
237	// Same page attribution as the rest of the session; the column pair was
238	// added by an upgrade routine, so tolerate its absence like the chat
239	// writer does.
240	$originating = MxChat_Session_Store::get($session_id, 'originating_page');
241	if (is_array($originating) && !empty($originating['url'])
242	&& $wpdb->get_var("SHOW COLUMNS FROM `$table` LIKE 'originating_page_url'")) {
243	$row['originating_page_url'] = $originating['url'];
244	$row['originating_page_title'] = isset($originating['title']) ? $originating['title'] : '';
245	}
246
247	$inserted = $wpdb->insert($table, $row);
248	if ($inserted === false) {
249	return 0;
250	}
251
252	$row_id = (int) $wpdb->insert_id;
253	self::flush_session_history_cache($session_id);
254
255	return $row_id;
256	}
257
258	/**
259	* Most recipients the Notification Email field will accept (plan 2f131a).
260	* A settings field is not a mailing list.
261	*/
262	const NOTIFICATION_EMAIL_MAX = 5;
263
264	/**
265	* Parse the Notification Email field into a list of recipients (plan 2f131a).
266	*
267	* THE TRAP THIS EXISTS TO CLOSE: sanitize_email() cannot be the validator for
268	* this field, because its output for the failing input is VALID. WordPress
269	* strips the separator and the surplus '@' and concatenates the remains:
270	*
271	* [email protected], [email protected] -> [email protected]
272	*
273	* and is_email() then returns true on that. So every guard in the plugin passed,
274	* the address was stored, the autosave ticked green, and both the new-session
275	* notification and the auto-emailed transcript went to a domain that does not
276	* exist — with no error anywhere. Validating the RAW part BEFORE sanitizing is
277	* the whole point; reversing those two lines silently restores the bug.
278	*
279	* All-or-nothing by design: if any entry is bad the caller must store NOTHING.
280	* A partial accept — keeping the good addresses and dropping the bad one — is
281	* the same defect in a new costume, because the owner still believes everyone
282	* on their list is being notified.
283	*
284	* @param mixed $raw Raw field value, exactly as submitted.
285	* @return array{emails: string[], error: string} Empty emails + empty error
286	* means the field was empty.
287	*/
288	public static function parse_notification_emails($raw) {
289	$out = array('emails' => array(), 'error' => '');
290
291	if (!is_scalar($raw)) {
292	$out['error'] = __('The notification email could not be read.', 'mxchat');
293	return $out;
294	}
295
296	$raw = trim((string) $raw);
297	if ($raw === '') {
298	return $out; // genuinely empty — the caller falls back to admin_email
299	}
300
301	$seen = array();
302	foreach (preg_split('/[,;]/', $raw) as $part) {
303	$part = trim($part);
304	if ($part === '') {
305	// A trailing or doubled separator carries no address, so skipping it
306	// cannot silently drop a recipient. This is the ONLY thing tolerated.
307	continue;
308	}
309
310	// RAW first. See the note above — order is load-bearing.
311	$clean = is_email($part) ? sanitize_email($part) : '';
312	if ($clean === '' \|\| !is_email($clean)) {
313	return array(
314	'emails' => array(),
315	'error' => sprintf(
316	/* translators: %s: the email address the owner typed. */
317	__('"%s" is not a valid email address, so nothing was saved. Separate multiple addresses with a comma.', 'mxchat'),
318	esc_html($part)
319	),
320	);
321	}
322
323	$key = strtolower($clean);
324	if (isset($seen[$key])) {
325	continue; // same address twice would simply mail them twice
326	}
327	$seen[$key] = true;
328	$out['emails'][] = $clean;
329	}
330
331	if (count($out['emails']) > self::NOTIFICATION_EMAIL_MAX) {
332	return array(
333	'emails' => array(),
334	'error' => sprintf(
335	/* translators: %d: maximum number of notification recipients. */
336	__('Enter at most %d email addresses, separated by commas.', 'mxchat'),
337	self::NOTIFICATION_EMAIL_MAX
338	),
339	);
340	}
341
342	return $out;
343	}
344
345	/**
346	* The stored recipient list, ready to hand to wp_mail() (plan 2f131a).
347	*
348	* Fallback rule, and it is narrow on purpose: an EMPTY field falls back to the
349	* site admin address, because that is the documented behaviour and an owner who
350	* never filled the field in still wants their notifications. A field holding
351	* something unusable does NOT fall back — it sends nowhere, exactly as before
352	* this plan. Falling back on bad input would mean a typo silently redirects a
353	* store's transcripts to a different mailbox than the one on screen.
354	*
355	* @param array\|null $options mxchat_transcripts_options, or null to read it.
356	* @return string[] Recipients; empty means do not send.
357	*/
358	public static function notification_recipients($options = null) {
359	if (!is_array($options)) {
360	$options = get_option('mxchat_transcripts_options', array());
361	if (!is_array($options)) {
362	$options = array();
363	}
364	}
365
366	$raw = isset($options['mxchat_notification_email']) ? $options['mxchat_notification_email'] : '';
367	$raw = is_scalar($raw) ? trim((string) $raw) : '';
368
369	if ($raw === '') {
370	$admin = get_option('admin_email');
371	return is_email($admin) ? array($admin) : array();
372	}
373
374	$parsed = self::parse_notification_emails($raw);
375	return $parsed['error'] === '' ? $parsed['emails'] : array();
376	}
377
378	/**
379	* Centralized embedding model registry. Single source of truth for dimensions
380	* and provider, so model-switch protection logic doesn't drift across files.
381	*/
382	public static function embedding_model_registry() {
383	return array(
384	'text-embedding-ada-002' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'Ada 2'),
385	'text-embedding-3-small' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'TE3 Small'),
386	'text-embedding-3-large' => array('dims' => 3072, 'provider' => 'openai', 'label' => 'TE3 Large'),
387	'voyage-3-large' => array('dims' => 2048, 'provider' => 'voyage', 'label' => 'Voyage-3 Large'),
388	'gemini-embedding-001' => array('dims' => 1536, 'provider' => 'gemini', 'label' => 'Gemini Embedding'),
389	);
390	}
391
392	public static function embedding_model_dimensions($model) {
393	$registry = self::embedding_model_registry();
394	return isset($registry[$model]) ? (int) $registry[$model]['dims'] : 0;
395	}
396
397	public static function embedding_model_label($model) {
398	if (is_string($model) && strpos($model, 'custom:') === 0) {
399	/* translators: %s: the embedding model name configured on the custom provider */
400	return sprintf(__('%s (custom provider)', 'mxchat'), substr($model, 7));
401	}
402	$registry = self::embedding_model_registry();
403	return isset($registry[$model]) ? $registry[$model]['label'] : $model;
404	}
405
406	/**
407	* Returns the model that was last used to actually write embeddings into the
408	* KB. Differs from the user-selected setting once a switch has happened but
409	* no re-embed has occurred yet — that's the mismatch state we warn about.
410	*/
411	public static function get_active_embedding_model() {
412	return get_option('mxchat_active_embedding_model', '');
413	}
414
415	/**
416	* Stamp the model that produced the most recent successful embedding. Called
417	* from generate_embedding() right after the API responds with a valid vector.
418	*/
419	public static function stamp_active_embedding_model($model) {
420	if (!empty($model) && $model !== self::get_active_embedding_model()) {
421	update_option('mxchat_active_embedding_model', $model, false);
422	}
423	}
424
425	/**
426	* The model name the custom-provider embedding path will send, mirroring the
427	* fallback chain the request itself uses: dedicated custom embedding model,
428	* else the custom chat model, else 'default'. Single source shared by
429	* generate_embedding_custom() and the mismatch-warning "selected" side so the
430	* two can never drift (plan ae02cb).
431	*/
432	public static function resolve_custom_embedding_model($options) {
433	if (isset($options['custom_provider_embedding_model']) && trim((string) $options['custom_provider_embedding_model']) !== '') {
434	return trim((string) $options['custom_provider_embedding_model']);
435	}
436	if (isset($options['custom_provider_model']) && trim((string) $options['custom_provider_model']) !== '') {
437	return trim((string) $options['custom_provider_model']);
438	}
439	return 'default';
440	}
441
442	/**
443	* The EFFECTIVE selected embedding model — what the next embed will actually
444	* use. With custom-provider embeddings on this is the custom identity in the
445	* same 'custom:<model>' form stamp_active_embedding_model() records, not the
446	* inert standard dropdown value. Mismatch-warning comparisons must read this,
447	* never $options['embedding_model'] directly — the dropdown cannot be
448	* deselected, so reading it raw flags every correctly-configured custom setup.
449	*/
450	public static function get_selected_embedding_model($options = null) {
451	if (!is_array($options)) {
452	$options = get_option('mxchat_options', array());
453	}
454	if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
455	return 'custom:' . self::resolve_custom_embedding_model($options);
456	}
457	return $options['embedding_model'] ?? '';
458	}
459
460	/**
461	* Dimension of a STANDARD embedding model, from the dropdown and its two
462	* adjustable-size settings. Says nothing about a Custom Provider model — ask
463	* expected_embedding_dimension() for the model the site actually embeds with.
464	*/
465	public static function standard_embedding_dimension($options = null) {
466	if (!is_array($options)) {
467	$options = get_option('mxchat_options', array());
468	$options = is_array($options) ? $options : array();
469	}
470	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
471	$model_dimensions = array(
472	'text-embedding-ada-002' => 1536,
473	'text-embedding-3-small' => 1536,
474	'text-embedding-3-large' => 3072,
475	'voyage-2' => 1024,
476	'voyage-large-2' => 1536,
477	'voyage-3-large' => 2048,
478	'gemini-embedding-001' => 1536,
479	);
480	if (strpos($selected_model, 'voyage-3-large') === 0) {
481	return intval($options['voyage_output_dimension'] ?? 2048);
482	}
483	if (strpos($selected_model, 'gemini-embedding') === 0) {
484	return intval($options['gemini_output_dimension'] ?? 1536);
485	}
486	return $model_dimensions[$selected_model] ?? 1536;
487	}
488
489	/**
490	* Dimension of the vectors the NEXT embed will produce — the one place that
491	* answers it (plan 418afd). With Custom Provider embeddings on, the standard
492	* dropdown is disabled and says nothing about the custom model, so the number
493	* is read from the model itself: one short probe embed, remembered per base
494	* URL + model. Three copies of the dropdown table used to answer instead, and
495	* a custom model of another size got a wrong-sized index and wrong-sized
496	* listing vectors.
497	*
498	* @param array\|null $options mxchat_options (or a bot's merged options).
499	* @param bool $force Probe again even when a recent probe failed.
500	* @return int\|WP_Error WP_Error only when a custom model's size could not be read.
501	*/
502	public static function expected_embedding_dimension($options = null, $force = false) {
503	if (!is_array($options)) {
504	$options = get_option('mxchat_options', array());
505	$options = is_array($options) ? $options : array();
506	}
507	if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
508	return self::custom_embedding_dimension($options, $force);
509	}
510	return self::standard_embedding_dimension($options);
511	}
512
513	/**
514	* A Custom Provider embedding model's dimension, by asking it once. The
515	* answer is stored against the base URL + model, so changing either asks
516	* again. A failure is remembered for five minutes so a dead endpoint does not
517	* hold up every admin page that prints the number; $force skips that memory.
518	*
519	* @return int\|WP_Error
520	*/
521	public static function custom_embedding_dimension($options, $force = false) {
522	$model = self::resolve_custom_embedding_model($options);
523	$base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
524	$key = md5(strtolower($base_url) . '\|' . $model);
525
526	$known = get_option('mxchat_custom_embedding_dimension', array());
527	if (is_array($known) && ($known['key'] ?? '') === $key && (int) ($known['dimension'] ?? 0) > 0) {
528	return (int) $known['dimension'];
529	}
530
531	$failed = get_transient('mxchat_custom_embedding_dimension_failed');
532	if (!$force && is_array($failed) && ($failed['key'] ?? '') === $key) {
533	return new WP_Error('embedding_dimension_unknown', (string) ($failed['reason'] ?? ''));
534	}
535
536	$vector = self::generate_embedding_custom('MxChat embedding dimension check', $options);
537	if (!is_array($vector) \|\| count($vector) === 0) {
538	$reason = (is_string($vector) && $vector !== '') ? $vector : __('The custom provider returned no embedding.', 'mxchat');
539	set_transient('mxchat_custom_embedding_dimension_failed', array('key' => $key, 'reason' => $reason), 5 * MINUTE_IN_SECONDS);
540	return new WP_Error('embedding_dimension_unknown', $reason);
541	}
542
543	update_option('mxchat_custom_embedding_dimension', array(
544	'key' => $key,
545	'model' => 'custom:' . $model,
546	'dimension' => count($vector),
547	), false);
548	delete_transient('mxchat_custom_embedding_dimension_failed');
549	return count($vector);
550	}
551
552	/**
553	* Extract the 11-character YouTube video ID from a URL, or '' if the URL is
554	* not a single-video YouTube link. Single source of truth for both the KB
555	* ingestion side and the chat render side — do not duplicate this parsing.
556	* Channel, playlist, and search URLs deliberately return '' (only a URL that
557	* identifies one video can be embedded).
558	*/
559	public static function parse_youtube_id($url) {
560	if (!is_string($url) \|\| $url === '' \|\| !preg_match('#^https?://#i', $url)) {
561	return '';
562	}
563	$host = strtolower((string) wp_parse_url($url, PHP_URL_HOST));
564	$host = preg_replace('/^(www\|m)\./', '', $host);
565	$path = (string) wp_parse_url($url, PHP_URL_PATH);
566	$id = '';
567	if ($host === 'youtu.be') {
568	$segments = explode('/', ltrim($path, '/'));
569	$id = $segments[0] ?? '';
570	} elseif (in_array($host, array('youtube.com', 'youtube-nocookie.com'), true)) {
571	if (preg_match('#^/(?:shorts\|embed\|live\|v)/([A-Za-z0-9_-]+)#', $path, $m)) {
572	$id = $m[1];
573	} elseif ($path === '/watch') {
574	parse_str((string) wp_parse_url($url, PHP_URL_QUERY), $query_vars);
575	$id = isset($query_vars['v']) ? (string) $query_vars['v'] : '';
576	}
577	}
578	$id = preg_replace('/[^A-Za-z0-9_-]/', '', (string) $id);
579	return (strlen($id) === 11) ? $id : '';
580	}
581
582	/**
583	* plan-mxchat-20260813-f52492 — video-card gating.
584	*
585	* Two standalone options (NOT mxchat_options — they skip the sanitize/autosave
586	* traps entirely), read here so the gate in the integrator and the fields on
587	* Knowledge -> Chunking & Retrieval can never disagree about a default.
588	*
589	* Master switch. Default ON: the card is existing behavior, and this is an
590	* opt-out for owners who never want one, not a new feature to opt into.
591	*/
592	public static function video_embed_enabled() {
593	return get_option('mxchat_video_embed_enabled', 'on') === 'on';
594	}
595
596	/**
597	* The video card's OWN confidence floor, as a 0-1 cosine — deliberately not
598	* the site-wide Similarity Threshold (default 35). "Good enough to quote in
599	* the answer" and "good enough to put a video on screen" are different
600	* questions: retrieval is allowed to be generous because the model still
601	* decides what to say, whereas the card is asserted to the visitor with no
602	* such filter. Stored as an int percentage to match the site-wide slider's
603	* convention; the default (55) sits above it on purpose.
604	*
605	* MXCHAT_VIDEO_EMBED_THRESHOLD_DEFAULT is the single source of that number.
606	*/
607	public static function video_embed_threshold() {
608	$stored = get_option('mxchat_video_embed_threshold', null);
609	$percent = ($stored === null \|\| $stored === '')
610	? MXCHAT_VIDEO_EMBED_THRESHOLD_DEFAULT
611	: (int) $stored;
612	if ($percent < 0) { $percent = 0; }
613	if ($percent > 100) { $percent = 100; }
614	// Cast: PHP evaluates 100/100 to int(1), so an unclamped return type would
615	// vary with the stored value. Callers compare against a cosine — keep it float.
616	return (float) $percent / 100;
617	}
618
619	/**
620	* UPDATED: Submit or update content (and its embedding) in the database.
621	* Stores in Pinecone if enabled, otherwise stores in WordPress DB.
622	*
623	* @param string $content The content to be embedded.
624	* @param string $source_url The source URL of the content.
625	* @param string $api_key The API key used for generating embeddings.
626	* @param string $vector_id Optional vector ID for Pinecone (if not provided, will use md5 of URL)
627	* @param string $bot_id The bot ID for multi-bot support
628	* @param string $content_type The type of content (post, page, pdf, url, manual, product, etc.)
629	* @return bool\|WP_Error True on success, WP_Error on failure
630	*/
631	public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content', $title = '') {
632	global $wpdb;
633	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
634
635	//error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ', Type: ' . $content_type . ')');
636	//error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
637
638	// Sanitize the source URL. Internal identities (mxchat:// manual docs,
639	// upload:// file uploads) are NOT web URLs — esc_url_raw EMPTIES them
640	// because its protocol list is statically cached and effectively
641	// unfilterable (measured, plan 945406 / 0485e5) — sanitize those as text
642	// so upserts stay keyed to a stable identity across re-imports.
643	if (preg_match('#^(mxchat\|upload)://#i', $source_url)) {
644	$source_url = sanitize_text_field($source_url);
645	} else {
646	$source_url = esc_url_raw($source_url);
647	}
648
649	// Sanitize content_type
650	$content_type = sanitize_key($content_type);
651	if (empty($content_type)) {
652	$content_type = 'content'; // Fallback for backwards compatibility
653	}
654
655	// Page title captured by the importer (plan d9ee66), registered under the
656	// SANITIZED url so the storage writers below find it by the same key.
657	if (is_string($title) && trim($title) !== '') {
658	self::set_source_title($source_url, $title);
659	}
660
661	// Just ensure UTF-8 validity without aggressive escaping
662	$safe_content = wp_check_invalid_utf8($content);
663	// Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D)
664	$safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
665
666	// Check if chunking should be applied
667	$chunker = MxChat_Chunker::from_settings();
668	if ($chunker->should_chunk($safe_content)) {
669	//error_log('[MXCHAT-DB] Content exceeds chunk threshold, using chunked submission');
670	$chunk_result = self::submit_chunked_content($safe_content, $source_url, $api_key, $bot_id, $content_type, $chunker);
671	// Vector Store mirror gets the entry WHOLE — one file per KB entry,
672	// OpenAI chunks server-side; local chunking is our own embedding
673	// concern and never reaches the store (plan 15b5c6).
674	if ($chunk_result === true && class_exists('MxChat_Vectorstore_Manager')) {
675	MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type);
676	}
677	return $chunk_result;
678	}
679
680	// UPDATED: Generate the embedding using bot-specific configuration
681	$embedding_vector = self::generate_embedding($content, $api_key, $bot_id);
682
683	if (!is_array($embedding_vector)) {
684	// Surface the provider's real reason instead of a fixed string (4a7c0a).
685	$reason = is_wp_error($embedding_vector)
686	? $embedding_vector->get_error_message()
687	: 'Failed to generate embedding for content';
688	return new WP_Error('embedding_failed', $reason);
689	}
690
691	//error_log('[MXCHAT-DB] Embedding generated successfully');
692
693	// UPDATED: Check if Pinecone is enabled for this specific bot
694	if (self::is_pinecone_enabled_for_bot($bot_id)) {
695	//error_log('[MXCHAT-DB] Pinecone is enabled for bot ' . $bot_id . ' - using Pinecone storage');
696	// Store in Pinecone only
697	$result = self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id, $content_type);
698	// If this URL previously stored as CHUNKED content and the new content fits in a
699	// single vector, the upsert above only overwrote the base id — the old
700	// md5(url)_chunk_N vectors would keep serving the stale text. Sweep them.
701	// Only for a real source_url: chunk ids derive from it, and md5('') is shared
702	// by legacy URL-less entries so a blind sweep there could hit other entries.
703	if ($result === true && !empty($source_url)) {
704	self::cleanup_pinecone_chunk_stragglers($source_url, $bot_id);
705	}
706	if ($result === true && class_exists('MxChat_Vectorstore_Manager')) {
707	MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type);
708	}
709	return $result;
710	} else {
711	//error_log('[MXCHAT-DB] Pinecone not enabled for bot ' . $bot_id . ' - using WordPress storage');
712	// Store in WordPress database only
713	$embedding_vector_serialized = maybe_serialize($embedding_vector);
714	$result = self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type);
715	if ($result === true && class_exists('MxChat_Vectorstore_Manager')) {
716	MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type);
717	}
718	return $result;
719	}
720	}
721
722	/* -------------------------------------------------------------------------
723	* Source titles (3.2.23, plan d9ee66)
724	*
725	* A knowledge entry imported from a URL or sitemap remembers the page's own
726	* title, so an add-on that names a source (AI Search's source cards) can show
727	* "About Us" instead of a guess from the slug ("Aboutus"). Stored in the
728	* `title` column of the WP knowledge table and as `title` metadata on Pinecone
729	* vectors / documents. The importer hands the title to submit_content_to_db(),
730	* which registers it under the source URL; the writers look it up by that key.
731	* ---------------------------------------------------------------------- */
732
733	/** @var array<string,string> source_url => title, for the current request */
734	private static $source_titles = array();
735
736	public static function set_source_title($source_url, $title) {
737	$key = trim((string) $source_url);
738	$title = self::clean_title($title);
739	if ($key === '' \|\| $title === '') {
740	return;
741	}
742	self::$source_titles[$key] = $title;
743	}
744
745	public static function source_title_for($source_url) {
746	$key = trim((string) $source_url);
747	return ($key !== '' && isset(self::$source_titles[$key])) ? self::$source_titles[$key] : '';
748	}
749
750	/** Entities decoded, tags gone, whitespace collapsed, capped at the column width. */
751	public static function clean_title($title) {
752	$title = html_entity_decode((string) $title, ENT_QUOTES \| ENT_HTML5, 'UTF-8');
753	$title = wp_strip_all_tags($title);
754	$title = trim((string) preg_replace('/\s+/u', ' ', $title));
755	if (mb_strlen($title, 'UTF-8') > 255) {
756	$title = rtrim(mb_substr($title, 0, 255, 'UTF-8'));
757	}
758	return $title;
759	}
760
761	/**
762	* The page's own title from fetched HTML: og:title, else <title> with a
763	* trailing "\| Site Name" style suffix removed, else the first <h1>. Returns ''
764	* when nothing usable is there, so the caller keeps its own fallback.
765	*/
766	public static function extract_page_title($html) {
767	$html = (string) $html;
768	if ($html === '') {
769	return '';
770	}
771	$head = substr($html, 0, 200000);
772	$site = '';
773	if (preg_match('/<meta\s+[^>]property=["\']og:site_name["\'][^>]content=["\']([^"\']{1,120})["\']/i', $head, $m)
774	\|\| preg_match('/<meta\s+[^>]content=["\']([^"\']{1,120})["\'][^>]property=["\']og:site_name["\']/i', $head, $m)) {
775	$site = self::clean_title($m[1]);
776	}
777	$candidate = '';
778	if (preg_match('/<meta\s+[^>]property=["\']og:title["\'][^>]content=["\']([^"\']{1,500})["\']/i', $head, $m)
779	\|\| preg_match('/<meta\s+[^>]content=["\']([^"\']{1,500})["\'][^>]property=["\']og:title["\']/i', $head, $m)) {
780	$candidate = self::clean_title($m[1]);
781	}
782	if ($candidate === '' && preg_match('/<title[^>]>(.?)<\/title>/is', $head, $m)) {
783	$candidate = self::clean_title($m[1]);
784	}
785	if ($candidate !== '') {
786	$candidate = self::strip_site_suffix($candidate, $site);
787	}
788	if ($candidate === '' && preg_match('/<h1[^>]>(.?)<\/h1>/is', $html, $m)) {
789	$candidate = self::clean_title($m[1]);
790	}
791	return $candidate;
792	}
793
794	/**
795	* "About Us \| Just Facts" -> "About Us". Only a TRAILING segment after a spaced
796	* separator goes, only when what is left is a real title (3+ characters) and the
797	* removed part is short enough to be a site name; a known og:site_name wins.
798	* Byte offsets throughout: every separator starts with an ASCII space, so a cut
799	* there never lands inside a multibyte character.
800	*/
801	private static function strip_site_suffix($title, $site_name = '') {
802	$seps = array(' \| ', ' – ', ' — ', ' - ', ' :: ', ' · ', ' » ', ' • ');
803	if ($site_name !== '') {
804	foreach ($seps as $sep) {
805	$tail = $sep . $site_name;
806	if (strlen($title) > strlen($tail) && substr($title, -strlen($tail)) === $tail) {
807	return trim(substr($title, 0, strlen($title) - strlen($tail)));
808	}
809	}
810	}
811	$best_pos = -1;
812	$best_sep = '';
813	foreach ($seps as $sep) {
814	$pos = strrpos($title, $sep);
815	if ($pos !== false && $pos > $best_pos) {
816	$best_pos = $pos;
817	$best_sep = $sep;
818	}
819	}
820	if ($best_pos > 0) {
821	$left = trim(substr($title, 0, $best_pos));
822	$right = trim(substr($title, $best_pos + strlen($best_sep)));
823	if (mb_strlen($left, 'UTF-8') >= 3 && $right !== '' && mb_strlen($right, 'UTF-8') <= 60) {
824	return $left;
825	}
826	}
827	return $title;
828	}
829
830	/**
831	* Make sure the knowledge table has the `title` column (3.2.23, plan d9ee66).
832	* Activation adds it; this is the safety net for a copy updated in place, and
833	* the one place add-ons ask "may I read/write title?". A positive answer is
834	* remembered in an option, so after the first time this costs one option read.
835	*/
836	public static function ensure_kb_title_column() {
837	global $wpdb;
838	static $ready = null;
839	if ($ready !== null) {
840	return $ready;
841	}
842	if (get_option('mxchat_kb_title_column') === '1') {
843	$ready = true;
844	return true;
845	}
846	$table = $wpdb->prefix . 'mxchat_system_prompt_content';
847	$cols = $wpdb->get_col("SHOW COLUMNS FROM {$table}", 0);
848	if (empty($cols)) {
849	$ready = false;
850	return false;
851	}
852	if (!in_array('title', $cols, true)) {
853	$wpdb->query("ALTER TABLE {$table} ADD COLUMN title VARCHAR(255) DEFAULT NULL AFTER source_url");
854	$cols = $wpdb->get_col("SHOW COLUMNS FROM {$table}", 0);
855	}
856	$ready = in_array('title', $cols, true);
857	if ($ready) {
858	update_option('mxchat_kb_title_column', '1', false);
859	}
860	return $ready;
861	}
862
863	/**
864	* UPDATED: Check if Pinecone is enabled and properly configured for a specific bot
865	*/
866	private static function is_pinecone_enabled_for_bot($bot_id = 'default') {
867	// For default bot or when multi-bot is not active, use original method
868	if ($bot_id === 'default' \|\| !class_exists('MxChat_Multi_Bot_Manager')) {
869	return self::is_pinecone_enabled();
870	}
871
872	// Get bot-specific Pinecone configuration
873	$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
874
875	if (empty($bot_pinecone_config)) {
876	// Fallback to default configuration
877	return self::is_pinecone_enabled();
878	}
879
880	$enabled_check = !empty($bot_pinecone_config['use_pinecone']) && $bot_pinecone_config['use_pinecone'];
881	$api_key_check = !empty($bot_pinecone_config['api_key']);
882	$host_check = !empty($bot_pinecone_config['host']);
883
884	return $enabled_check && $api_key_check && $host_check;
885	}
886
887	/**
888	* Check if Pinecone is enabled and properly configured (original method for default bot)
889	*/
890	private static function is_pinecone_enabled() {
891	$pinecone_options = get_option('mxchat_pinecone_addon_options');
892
893	if (empty($pinecone_options)) {
894	return false;
895	}
896
897	$enabled_check = !empty($pinecone_options['mxchat_use_pinecone']) && $pinecone_options['mxchat_use_pinecone'] !== '0';
898	$api_key_check = !empty($pinecone_options['mxchat_pinecone_api_key']);
899	$host_check = !empty($pinecone_options['mxchat_pinecone_host']);
900
901	return $enabled_check && $api_key_check && $host_check;
902	}
903
904	/**
905	* UPDATED: Store content in Pinecone only with bot support
906	*/
907	private static function store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
908	//error_log('[MXCHAT-PINECONE] ===== Using Pinecone-only storage for bot ' . $bot_id . ' =====');
909
910	// Get bot-specific Pinecone configuration
911	if ($bot_id === 'default' \|\| !class_exists('MxChat_Multi_Bot_Manager')) {
912	$pinecone_options = get_option('mxchat_pinecone_addon_options');
913	$api_key = $pinecone_options['mxchat_pinecone_api_key'];
914	$environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
915	$index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
916	$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
917	} else {
918	$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
919	if (empty($bot_pinecone_config)) {
920	// Fallback to default configuration
921	$pinecone_options = get_option('mxchat_pinecone_addon_options');
922	$api_key = $pinecone_options['mxchat_pinecone_api_key'];
923	$environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
924	$index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
925	$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
926	} else {
927	$api_key = $bot_pinecone_config['api_key'];
928	$environment = ''; // Not used in new Pinecone API
929	$index_name = ''; // Not used in new Pinecone API
930	$namespace = $bot_pinecone_config['namespace'] ?? '';
931	}
932	}
933
934	$result = self::store_in_pinecone_main(
935	$embedding_vector,
936	$content,
937	$source_url,
938	$api_key,
939	$environment,
940	$index_name,
941	$vector_id,
942	$bot_id,
943	$namespace,
944	$content_type
945	);
946
947	if (is_wp_error($result)) {
948	//error_log('[MXCHAT-PINECONE] Pinecone storage failed for bot ' . $bot_id . ': ' . $result->get_error_message());
949	return $result;
950	}
951
952	//error_log('[MXCHAT-PINECONE] Pinecone storage completed successfully for bot ' . $bot_id);
953	return true;
954	}
955
956	/**
957	* Store content in WordPress database with progressive fallback
958	* UPDATED 2.5.6: Now includes content_type parameter
959	*/
960	private static function store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type = 'content') {
961	global $wpdb;
962
963	//error_log('[MXCHAT-DB] ===== Using WordPress-only storage =====');
964
965	// Sanitize content_type
966	$content_type = sanitize_key($content_type);
967	if (empty($content_type)) {
968	$content_type = 'content'; // Fallback for backwards compatibility
969	}
970
971	// ===== FIXED: Generate unique identifier for manual content =====
972	$original_source_url = $source_url;
973	// Check if this is truly manual content (no URL at all) vs a real URL that filter_var rejects
974	// filter_var(FILTER_VALIDATE_URL) rejects valid URLs with encoded chars, non-ASCII, fragments, etc.
975	// Use a looser check: if it starts with http(s):// or has a scheme, it's a URL
976	$has_url_scheme = !empty($source_url) && preg_match('#^https?://#i', $source_url);
977	// upload:// identities (admin document/PDF uploads, plan 0485e5) are stable
978	// and deduplicable — treat them like URLs so a re-upload UPDATES the row
979	// instead of minting a fresh manual identity (which would duplicate).
980	$has_stable_identity = $has_url_scheme \|\| (!empty($source_url) && preg_match('#^upload://#i', $source_url));
981	// Treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
982	$is_legacy_mxchat_url = $has_url_scheme && strpos($source_url, 'mxchat.ai') !== false;
983	$is_manual_content = empty($source_url) \|\| $source_url === '' \|\| !$has_stable_identity \|\| $is_legacy_mxchat_url;
984
985	if ($is_manual_content) {
986	// Generate unique identifier for manual content to prevent overwrites
987	$source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
988	//error_log('[MXCHAT-DB] Generated unique ID for manual content: ' . $source_url);
989	}
990
991	// Only check for duplicates if we have a valid source URL (not manual content)
992	$existing_id = null;
993	if (!$is_manual_content) {
994	$existing_id = $wpdb->get_var(
995	$wpdb->prepare(
996	"SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
997	$source_url
998	)
999	);
1000	//error_log('[MXCHAT-DB] Checked for existing URL, found ID: ' . ($existing_id ?: 'none'));
1001	} else {
1002	//error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)');
1003	}
1004	// ===== END FIX =====
1005
1006	// Page title captured at import (plan d9ee66): written only when the
1007	// importer supplied one, so a re-index without a title keeps the old value.
1008	$title_value = $is_manual_content ? '' : self::source_title_for($source_url);
1009	$title_ready = ($title_value !== '') && self::ensure_kb_title_column();
1010
1011	// Progressive fallback mechanism for problematic content
1012	$attempt = 1;
1013	$max_attempts = 3;
1014	$current_content = $safe_content;
1015	$result = false;
1016
1017	while ($attempt <= $max_attempts && $result === false) {
1018	try {
1019	if ($existing_id) {
1020	//error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
1021
1022	// Update the existing row - UPDATED 2.5.6: Added content_type
1023	$row = array(
1024	'url' => $source_url,
1025	'article_content' => $current_content,
1026	'embedding_vector' => $embedding_vector_serialized,
1027	'source_url' => $source_url,
1028	'content_type' => $content_type,
1029	'timestamp' => current_time('mysql'),
1030	);
1031	$formats = array('%s','%s','%s','%s','%s','%s');
1032	if ($title_ready) {
1033	$row['title'] = $title_value;
1034	$formats[] = '%s';
1035	}
1036	$result = $wpdb->update($table_name, $row, array('id' => $existing_id), $formats, array('%d'));
1037	} else {
1038	//error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
1039	//error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
1040
1041	// Insert a new row - UPDATED 2.5.6: Added content_type
1042	$row = array(
1043	'url' => $source_url, // Now unique for manual content
1044	'article_content' => $current_content,
1045	'embedding_vector' => $embedding_vector_serialized,
1046	'source_url' => $source_url, // Now unique for manual content
1047	'content_type' => $content_type,
1048	'timestamp' => current_time('mysql'),
1049	);
1050	$formats = array('%s','%s','%s','%s','%s','%s');
1051	if ($title_ready) {
1052	$row['title'] = $title_value;
1053	$formats[] = '%s';
1054	}
1055	$result = $wpdb->insert($table_name, $row, $formats);
1056	}
1057
1058	if ($result === false) {
1059	//error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . ')');
1060	//error_log('[MXCHAT-DB] MySQL Error: ' . $wpdb->last_error);
1061	//error_log('[MXCHAT-DB] MySQL Error Number: ' . $wpdb->last_errno);
1062	//error_log('[MXCHAT-DB] Last Query: ' . substr($wpdb->last_query, 0, 500));
1063	//error_log('[MXCHAT-DB] Content length: ' . strlen($current_content) . ' bytes');
1064	//error_log('[MXCHAT-DB] Embedding vector length: ' . strlen($embedding_vector_serialized) . ' bytes');
1065
1066	// Progressively apply more aggressive sanitization on failure
1067	if ($attempt === 1) {
1068	// First fallback: Use a more aggressive character filter and shorten
1069	$current_content = preg_replace('/[^\p{L}\p{N}\s.,;:!?()-]/u', '', $current_content);
1070	$current_content = substr($current_content, 0, 50000);
1071	} else if ($attempt === 2) {
1072	// Second fallback: Keep only alphanumeric and basic punctuation, shorten further
1073	$current_content = preg_replace('/[^a-zA-Z0-9\s.,;:!?()-]/u', '', $current_content);
1074	$current_content = substr($current_content, 0, 30000);
1075	}
1076
1077	$attempt++;
1078	}
1079	} catch (Exception $e) {
1080	//error_log('[MXCHAT-DB] Exception during database operation: ' . $e->getMessage());
1081	$attempt++;
1082	}
1083	}
1084
1085	if ($result === false) {
1086	//error_log('[MXCHAT-DB] All database operation attempts failed');
1087	//error_log('[MXCHAT-DB] Final MySQL Error: ' . $wpdb->last_error);
1088
1089	$detailed_error = sprintf(
1090	'Failed to store content in WordPress database after %d attempts. MySQL Error: %s (Error #%d). Content size: %d bytes, Embedding size: %d bytes',
1091	$max_attempts,
1092	$wpdb->last_error,
1093	$wpdb->last_errno,
1094	strlen($current_content),
1095	strlen($embedding_vector_serialized)
1096	);
1097
1098	return new WP_Error('database_failed', $detailed_error);
1099	}
1100
1101	//error_log('[MXCHAT-DB] WordPress database operation completed successfully (Attempt ' . ($attempt - 1) . ')');
1102	return true;
1103	}
1104
1105	/**
1106	* UPDATED: Store content in Pinecone database with bot support
1107	* UPDATED 2.5.6: Now accepts content_type parameter
1108	*/
1109	private static function store_in_pinecone_main($embedding_vector, $content, $url, $api_key, $environment, $index_name, $vector_id = null, $bot_id = 'default', $namespace = '', $content_type = 'content') {
1110	//error_log('[MXCHAT-PINECONE-MAIN] ===== Starting Pinecone storage for bot ' . $bot_id . ' =====');
1111
1112	// ===== UPDATED: Handle manual content with unique vector IDs =====
1113	if ($vector_id) {
1114	// Use provided vector ID
1115	//error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
1116	} elseif (!empty($url) && preg_match('#^(https?\|upload)://#i', $url)) {
1117	// For URLs — and stable upload:// identities (plan 0485e5) — use an
1118	// identity-derived ID so a re-import upserts the same vector.
1119	$vector_id = md5($url);
1120	//error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
1121	} else {
1122	// For manual content (empty/no URL scheme), generate unique ID
1123	$vector_id = 'manual_' . time() . '_' . substr(md5($content . microtime(true)), 0, 8);
1124	//error_log('[MXCHAT-PINECONE-MAIN] Generated unique vector ID for manual content: ' . $vector_id);
1125	}
1126	// ===== END UPDATE =====
1127
1128	// Get host from bot-specific config or fallback to default
1129	if ($bot_id === 'default' \|\| !class_exists('MxChat_Multi_Bot_Manager')) {
1130	$options = get_option('mxchat_pinecone_addon_options');
1131	$host = $options['mxchat_pinecone_host'] ?? '';
1132	} else {
1133	$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1134	if (!empty($bot_pinecone_config)) {
1135	$host = $bot_pinecone_config['host'] ?? '';
1136	} else {
1137	$options = get_option('mxchat_pinecone_addon_options');
1138	$host = $options['mxchat_pinecone_host'] ?? '';
1139	}
1140	}
1141
1142	//error_log('[MXCHAT-PINECONE-MAIN] Host: ' . $host);
1143	//error_log('[MXCHAT-PINECONE-MAIN] API key length: ' . strlen($api_key));
1144	//error_log('[MXCHAT-PINECONE-MAIN] Bot ID: ' . $bot_id);
1145	//error_log('[MXCHAT-PINECONE-MAIN] Namespace: ' . $namespace);
1146
1147	if (empty($host)) {
1148	//error_log('[MXCHAT-PINECONE-MAIN] ERROR: Host is empty');
1149	return new WP_Error('pinecone_config', 'Pinecone host is not configured. Please set the host in your bot settings.');
1150	}
1151
1152	// ===== UPDATED 2.5.6: Use passed content_type or determine from URL if not provided =====
1153	// Sanitize content_type
1154	$content_type = sanitize_key($content_type);
1155	if (empty($content_type)) {
1156	// Fallback to old detection logic for backwards compatibility
1157	$is_product = false;
1158	$content_type = 'manual'; // Default for manual content
1159
1160	if (!empty($url) && preg_match('#^https?://#i', $url)) {
1161	$is_product = (strpos($url, '/product/') !== false \|\| strpos($url, '/shop/') !== false);
1162	$content_type = $is_product ? 'product' : 'content';
1163	}
1164	}
1165
1166	//error_log('[MXCHAT-PINECONE-MAIN] Content type: ' . $content_type);
1167	// ===== END UPDATE =====
1168
1169	$api_endpoint = "https://{$host}/vectors/upsert";
1170	//error_log('[MXCHAT-PINECONE-MAIN] API endpoint: ' . $api_endpoint);
1171
1172	// UPDATED 2.5.6: Use provided content_type in metadata
1173	$metadata = array(
1174	'text' => $content,
1175	'source_url' => $url, // Can be empty for manual content
1176	'type' => $content_type, // Now supports: post, page, pdf, url, manual, product, etc.
1177	'last_updated' => time(),
1178	'created_at' => time(), // Add creation timestamp
1179	'bot_id' => $bot_id, // Add bot identification
1180	);
1181	$page_title = self::source_title_for($url);
1182	if ($page_title !== '') {
1183	$metadata['title'] = $page_title; // page title captured at import (plan d9ee66)
1184	}
1185
1186	// Document index (plan 362c31): same id, same metadata, through the
1187	// documents API. A site on the classic vector index never enters here.
1188	if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) {
1189	return MxChat_Pinecone_Documents::upsert_one($vector_id, $content, $embedding_vector, $metadata, array(
1190	'api_key' => $api_key,
1191	'host' => $host,
1192	'namespace' => $namespace,
1193	));
1194	}
1195
1196	$vector_data = array(
1197	'id' => $vector_id,
1198	'values' => $embedding_vector,
1199	'metadata' => $metadata
1200	);
1201
1202	$request_body = array(
1203	'vectors' => array($vector_data)
1204	);
1205
1206	// Add namespace if specified for multi-bot separation
1207	if (!empty($namespace)) {
1208	$request_body['namespace'] = $namespace;
1209	//error_log('[MXCHAT-PINECONE-MAIN] Using namespace: ' . $namespace);
1210	}
1211
1212	//error_log('[MXCHAT-PINECONE-MAIN] Request body prepared (embedding dimensions: ' . count($embedding_vector) . ')');
1213
1214	$response = wp_remote_post($api_endpoint, array(
1215	'headers' => array(
1216	'Api-Key' => $api_key,
1217	'accept' => 'application/json',
1218	'content-type' => 'application/json'
1219	),
1220	'body' => wp_json_encode($request_body),
1221	'timeout' => 30,
1222	'data_format' => 'body'
1223	));
1224
1225	if (is_wp_error($response)) {
1226	//error_log('[MXCHAT-PINECONE-MAIN] WordPress request error: ' . $response->get_error_message());
1227	return new WP_Error('pinecone_request', $response->get_error_message());
1228	}
1229
1230	$response_code = wp_remote_retrieve_response_code($response);
1231	//error_log('[MXCHAT-PINECONE-MAIN] Response code: ' . $response_code);
1232
1233	if ($response_code !== 200) {
1234	$body = wp_remote_retrieve_body($response);
1235	//error_log('[MXCHAT-PINECONE-MAIN] API error - Response body: ' . $body);
1236	return new WP_Error('pinecone_api', sprintf(
1237	'Pinecone API error (HTTP %d): %s',
1238	$response_code,
1239	$body
1240	));
1241	}
1242
1243	$response_body = wp_remote_retrieve_body($response);
1244	//error_log('[MXCHAT-PINECONE-MAIN] Success response: ' . $response_body);
1245	//error_log('[MXCHAT-PINECONE-MAIN] Successfully stored in Pinecone for bot ' . $bot_id);
1246	//error_log('[MXCHAT-PINECONE-MAIN] ===== Pinecone storage complete =====');
1247
1248	return true;
1249	}
1250
1251	/**
1252	* Caller-side pre-flight for KB ingestion: can an embedding request be made
1253	* with these options, and which API key should travel downstream?
1254	*
1255	* Custom-provider-aware — generate_embedding() below routes to the custom
1256	* endpoint FIRST and ignores the passed cloud key entirely when
1257	* custom_provider_for_embeddings is on, so on that branch the only real
1258	* requirement is a Base URL. Ingestion callers that gated on a cloud API key
1259	* were killing keyless custom-embeddings sites (local Ollama / LM Studio
1260	* class) before the embed layer could route (plan cbd5fd).
1261	*
1262	* NOTE: reads $options['embedding_model'] raw on purpose — this mirrors
1263	* generate_embedding()'s own routing read, NOT the mismatch-banner's
1264	* "selected" chain (get_selected_embedding_model). The helper must predict
1265	* what the very next embed call will do, byte-for-byte.
1266	*
1267	* Decision only — callers keep their own error-surfacing shape (admin-notice
1268	* transient + redirect, wp_send_json_error, WP_Error, silent return).
1269	*
1270	* @param array\|null $options Resolved options (bot-specific where the caller
1271	* has them); null loads the default bot's options.
1272	* @return array {
1273	* @type bool $ok Whether ingestion can proceed.
1274	* @type string $api_key Key to pass downstream ('' on the custom branch —
1275	* generate_embedding() ignores it there).
1276	* @type string $reason Human-readable blocker; '' when $ok.
1277	* @type string $provider Short provider label ('OpenAI', 'Voyage AI',
1278	* 'Google Gemini', 'Custom Provider').
1279	* }
1280	*/
1281	public static function embedding_preflight($options = null) {
1282	if (!is_array($options)) {
1283	$options = get_option('mxchat_options');
1284	$options = is_array($options) ? $options : array();
1285	}
1286
1287	// Custom branch mirrors generate_embedding()'s routing order (custom first).
1288	if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
1289	$base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
1290	if ($base_url === '') {
1291	return array(
1292	'ok' => false,
1293	'api_key' => '',
1294	// Same string generate_embedding_custom() returns for this state.
1295	'reason' => __('Custom provider Base URL is not configured.', 'mxchat'),
1296	'provider' => 'Custom Provider',
1297	);
1298	}
1299	return array('ok' => true, 'api_key' => '', 'reason' => '', 'provider' => 'Custom Provider');
1300	}
1301
1302	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
1303	if (strpos($selected_model, 'voyage') === 0) {
1304	$api_key = $options['voyage_api_key'] ?? '';
1305	$provider = 'Voyage AI';
1306	} elseif (strpos($selected_model, 'gemini-embedding') === 0) {
1307	$api_key = $options['gemini_api_key'] ?? '';
1308	$provider = 'Google Gemini';
1309	} else {
1310	$api_key = $options['api_key'] ?? '';
1311	$provider = 'OpenAI';
1312	}
1313
1314	if (empty($api_key)) {
1315	return array(
1316	'ok' => false,
1317	'api_key' => '',
1318	'reason' => sprintf(
1319	/* translators: %s: embedding provider name */
1320	__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
1321	$provider
1322	),
1323	'provider' => $provider,
1324	);
1325	}
1326
1327	return array('ok' => true, 'api_key' => $api_key, 'reason' => '', 'provider' => $provider);
1328	}
1329
1330	/**
1331	* Public QUERY-side entry point (plan 876edb). The chat pipeline's
1332	* MxChat_Integrator::mxchat_generate_embedding() adapter routes through here
1333	* so the query and index sides share ONE provider-routing implementation —
1334	* the same endpoints, request bodies, and stamping semantics. The Integrator
1335	* keeps its own error vocabulary by translating the WP_Error this returns
1336	* (see the structured error data on every failure path below).
1337	*
1338	* @param string $text The text to be embedded.
1339	* @param string $api_key Caller-resolved API key (per-bot on the query side).
1340	* @param string $bot_id The bot ID for multi-bot support.
1341	* @return array\|WP_Error The embedding vector, or WP_Error carrying the reason.
1342	*/
1343	public static function generate_query_embedding($text, $api_key, $bot_id = 'default') {
1344	return self::generate_embedding($text, $api_key, $bot_id);
1345	}
1346
1347	/**
1348	* UPDATED: Generate an embedding for the given text using bot-specific configuration.
1349	*
1350	* @param string $text The text to be embedded.
1351	* @param string $api_key The API key used for generating embeddings.
1352	* @param string $bot_id The bot ID for multi-bot support
1353	* @return array\|null The embedding vector or null on failure.
1354	*/
1355	private static function generate_embedding($text, $api_key, $bot_id = 'default') {
1356	// Get bot-specific options
1357	if ($bot_id === 'default' \|\| !class_exists('MxChat_Multi_Bot_Manager')) {
1358	$options = get_option('mxchat_options');
1359	} else {
1360	$bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
1361	$options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
1362	}
1363
1364	// Opt-in: when the custom provider is selected for embeddings, route the KB
1365	// INDEX side through the same custom endpoint the query side uses, so stored
1366	// vectors and query vectors come from the same model. Default-off behavior
1367	// below is untouched.
1368	if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
1369	$custom = self::generate_embedding_custom($text, $options);
1370	// The custom path already returns a human-readable error string —
1371	// carry it instead of collapsing to null (plan 4a7c0a). The 'custom'
1372	// branch marker lets the Integrator adapter map the string back onto
1373	// its own error codes (876edb).
1374	return is_array($custom) ? $custom : new WP_Error('embedding_failed', (string) $custom, array('branch' => 'custom'));
1375	}
1376
1377	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
1378
1379	// Determine endpoint and API key based on model
1380	if (strpos($selected_model, 'voyage') === 0) {
1381	$endpoint = 'https://api.voyageai.com/v1/embeddings';
1382	$api_key = $options['voyage_api_key'] ?? '';
1383	} elseif (strpos($selected_model, 'gemini-embedding') === 0) {
1384	$endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
1385	$api_key = $options['gemini_api_key'] ?? '';
1386	} else {
1387	$endpoint = 'https://api.openai.com/v1/embeddings';
1388	// Prefer the caller-resolved key when one was passed — the query side
1389	// resolves per-bot keys at its call sites (integrator adapter, 876edb).
1390	// Index callers pass the preflight key, which equals this options read,
1391	// so nothing changes for them.
1392	$api_key = !empty($api_key) ? $api_key : ($options['api_key'] ?? '');
1393	}
1394
1395	// Prepare request body based on provider
1396	if (strpos($selected_model, 'gemini-embedding') === 0) {
1397	// Gemini API format
1398	$request_body = [
1399	'model' => 'models/' . $selected_model,
1400	'content' => [
1401	'parts' => [
1402	['text' => $text]
1403	]
1404	],
1405	'outputDimensionality' => 1536
1406	];
1407
1408	// Prepare headers for Gemini (API key as query parameter)
1409	$endpoint .= '?key=' . $api_key;
1410	$headers = [
1411	'Content-Type' => 'application/json'
1412	];
1413	} else {
1414	// OpenAI/Voyage API format
1415	$request_body = [
1416	'input' => $text,
1417	'model' => $selected_model
1418	];
1419
1420	// Add output_dimension for voyage-3-large
1421	if ($selected_model === 'voyage-3-large') {
1422	$request_body['output_dimension'] = 2048;
1423	}
1424
1425	// Prepare headers for OpenAI/Voyage
1426	$headers = [
1427	'Content-Type' => 'application/json',
1428	'Authorization' => 'Bearer ' . $api_key
1429	];
1430	}
1431
1432	$args = [
1433	'body' => wp_json_encode($request_body),
1434	'headers' => $headers,
1435	'timeout' => 60,
1436	'redirection' => 5,
1437	'blocking' => true,
1438	'httpversion' => '1.0',
1439	'sslverify' => true,
1440	];
1441
1442	$response = wp_remote_post($endpoint, $args);
1443
1444	if (is_wp_error($response)) {
1445	$message = 'Embedding request failed (connection): ' . $response->get_error_message();
1446	if (class_exists('MxChat_Admin')) {
1447	MxChat_Admin::mxchat_log_debug('embedding_error', $message, array('model' => $selected_model, 'bot_id' => $bot_id));
1448	}
1449	return new WP_Error('embedding_failed', $message, array(
1450	'branch' => 'cloud',
1451	'kind' => 'connection',
1452	'reason' => $response->get_error_message(),
1453	'model' => $selected_model,
1454	));
1455	}
1456
1457	$response_body = json_decode(wp_remote_retrieve_body($response), true);
1458
1459	// Handle different response formats based on provider
1460	if (strpos($selected_model, 'gemini-embedding') === 0) {
1461	// Gemini API response format
1462	if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
1463	self::stamp_active_embedding_model($selected_model);
1464	return $response_body['embedding']['values'];
1465	} else {
1466	return self::embedding_failure_error($response, $selected_model, $api_key, $bot_id);
1467	}
1468	} else {
1469	// OpenAI/Voyage API response format
1470	if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
1471	self::stamp_active_embedding_model($selected_model);
1472	return $response_body['data'][0]['embedding'];
1473	} else {
1474	return self::embedding_failure_error($response, $selected_model, $api_key, $bot_id);
1475	}
1476	}
1477	}
1478
1479	/**
1480	* Build a WP_Error carrying the embedding provider's REAL failure reason,
1481	* and record it in the Debug Mode log. Previously every failure path
1482	* returned bare null, so customers saw only "Failed to generate embedding
1483	* for content" / "Failed to store any chunks" with no cause (plan 4a7c0a).
1484	*
1485	* The API key never appears in provider response bodies (it travels in the
1486	* request headers), but the reason is scrubbed for it anyway before it can
1487	* reach a notice or the debug log.
1488	*/
1489	private static function embedding_failure_error($response, $selected_model, $api_key, $bot_id) {
1490	$status = (int) wp_remote_retrieve_response_code($response);
1491	$raw = (string) wp_remote_retrieve_body($response);
1492	$decoded = json_decode($raw, true);
1493
1494	// Provider error shapes: OpenAI + Gemini use {"error":{"message":…}};
1495	// Voyage uses {"detail":…}.
1496	$reason = '';
1497	if (is_array($decoded)) {
1498	if (isset($decoded['error']['message']) && is_string($decoded['error']['message'])) {
1499	$reason = $decoded['error']['message'];
1500	} elseif (isset($decoded['detail']) && is_string($decoded['detail'])) {
1501	$reason = $decoded['detail'];
1502	}
1503	}
1504	if ($reason === '') {
1505	$reason = ($raw !== '') ? substr($raw, 0, 200) : 'empty or malformed response';
1506	}
1507	if (is_string($api_key) && $api_key !== '') {
1508	$reason = str_replace($api_key, '[redacted]', $reason);
1509	}
1510	$reason = substr($reason, 0, 300);
1511	$message = sprintf('Embedding failed (%s, HTTP %d): %s', $selected_model, $status, $reason);
1512
1513	if (class_exists('MxChat_Admin')) {
1514	MxChat_Admin::mxchat_log_debug('embedding_error', $message, array(
1515	'model' => $selected_model,
1516	'status' => $status,
1517	'bot_id' => $bot_id,
1518	));
1519	}
1520
1521	// Structured data so the Integrator's query-side adapter can rebuild its
1522	// typed error contract (auth/rate-limit/quota/invalid-response) without a
1523	// second transport implementation (876edb). Additive — message unchanged.
1524	return new WP_Error('embedding_failed', $message, array(
1525	'branch' => 'cloud',
1526	'status' => $status,
1527	'error_type' => (is_array($decoded) && isset($decoded['error']['type']) && is_string($decoded['error']['type'])) ? $decoded['error']['type'] : '',
1528	'reason' => $reason,
1529	'model' => $selected_model,
1530	));
1531	}
1532
1533	/**
1534	* Generate an embedding via a Custom (OpenAI-compatible) provider's /embeddings route.
1535	* Shared by every embedding entry point so the KNOWLEDGE-BASE INDEX side and the
1536	* QUERY side route through the same model when the opt-in
1537	* 'custom_provider_for_embeddings' setting is on. Mirrors the query-path logic in
1538	* MxChat_Integrator::mxchat_generate_embedding_custom() but takes an explicit
1539	* $options array so it is callable statically from utils + knowledge-manager.
1540	*
1541	* Returns a numeric array (the embedding vector) on success, or a human-readable
1542	* error string on failure (so callers expecting a string error, like the
1543	* knowledge-manager, can surface it directly; callers expecting array\|null wrap it).
1544	*
1545	* @param string $text Text to embed.
1546	* @param array $options The resolved mxchat options (must contain the custom_provider_* keys).
1547	* @return array\|string Embedding vector on success; error string on failure.
1548	*/
1549	public static function generate_embedding_custom($text, $options) {
1550	if (empty($text)) {
1551	return 'No text provided for embedding generation';
1552	}
1553
1554	$base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
1555	if (empty($base_url)) {
1556	return 'Custom provider Base URL is not configured.';
1557	}
1558
1559	$api_key = isset($options['custom_provider_api_key']) ? trim((string) $options['custom_provider_api_key']) : '';
1560	$auth_scheme = isset($options['custom_provider_auth_scheme']) ? $options['custom_provider_auth_scheme'] : 'bearer';
1561	$api_version = isset($options['custom_provider_api_version']) ? trim((string) $options['custom_provider_api_version']) : '';
1562
1563	// Embedding model: shared resolver (dedicated embedding model -> chat model
1564	// -> 'default') — the mismatch warning's "selected" side reads the same chain.
1565	$model = self::resolve_custom_embedding_model($options);
1566
1567	$embed_url = $base_url . '/embeddings';
1568	if (!empty($api_version)) {
1569	$embed_url .= (strpos($embed_url, '?') === false ? '?' : '&') . 'api-version=' . rawurlencode($api_version);
1570	}
1571
1572	$headers = ['Content-Type' => 'application/json'];
1573	if (!empty($api_key)) {
1574	if ($auth_scheme === 'api-key') {
1575	$headers['api-key'] = $api_key;
1576	} else {
1577	$headers['Authorization'] = 'Bearer ' . $api_key;
1578	}
1579	}
1580
1581	$response = wp_remote_post($embed_url, [
1582	'headers' => $headers,
1583	'body' => wp_json_encode(['input' => $text, 'model' => $model]),
1584	'timeout' => 60,
1585	]);
1586	if (is_wp_error($response)) {
1587	return self::log_custom_embedding_failure(
1588	'Connection error when generating embeddings (custom provider): ' . $response->get_error_message(),
1589	$model,
1590	$api_key
1591	);
1592	}
1593
1594	$status = wp_remote_retrieve_response_code($response);
1595	$body = json_decode(wp_remote_retrieve_body($response), true);
1596	if ($status !== 200) {
1597	$msg = isset($body['error']['message']) ? $body['error']['message'] : 'HTTP ' . $status;
1598	return self::log_custom_embedding_failure(
1599	'Custom embedding endpoint error: ' . $msg,
1600	$model,
1601	$api_key,
1602	(int) $status
1603	);
1604	}
1605	if (isset($body['data'][0]['embedding']) && is_array($body['data'][0]['embedding'])) {
1606	// Stamp the custom model identity so the active-embedding-model mismatch
1607	// warning reflects the real (custom) model rather than the built-in setting.
1608	self::stamp_active_embedding_model('custom:' . $model);
1609	return $body['data'][0]['embedding'];
1610	}
1611	return self::log_custom_embedding_failure('Invalid embedding response from custom provider.', $model, $api_key);
1612	}
1613
1614	/**
1615	* Record a custom-provider embedding failure in the Debug Mode log, then
1616	* return the message unchanged so callers keep their string-error contract.
1617	* The cloud branch has logged its failures since 4a7c0a; the custom branch
1618	* never did, so chat-side failures on Custom-provider installs were
1619	* invisible to Debug Mode despite the 3.2.18 readme saying otherwise
1620	* (plan 71e4b6). Same scrub-then-log shape as embedding_failure_error().
1621	*
1622	* @param string $message Human-readable failure (the caller's return value).
1623	* @param string $model Resolved custom embedding model.
1624	* @param string $api_key Scrubbed out of the logged message if it ever appears.
1625	* @param int $status HTTP status when one was received, 0 otherwise.
1626	* @return string The (scrubbed) message.
1627	*/
1628	private static function log_custom_embedding_failure($message, $model, $api_key, $status = 0) {
1629	if (is_string($api_key) && $api_key !== '') {
1630	$message = str_replace($api_key, '[redacted]', $message);
1631	}
1632
1633	if (class_exists('MxChat_Admin')) {
1634	$context = array('model' => 'custom:' . $model);
1635	if ($status > 0) {
1636	$context['status'] = $status;
1637	}
1638	MxChat_Admin::mxchat_log_debug('embedding_error', $message, $context);
1639	}
1640
1641	return $message;
1642	}
1643
1644	/**
1645	* Submit content as multiple chunks
1646	*
1647	* Splits large content into chunks, generates embeddings for each,
1648	* and stores them with chunk metadata for later reassembly.
1649	*
1650	* @param string $content The content to chunk and store
1651	* @param string $source_url The source URL
1652	* @param string $api_key The API key for embeddings
1653	* @param string $bot_id The bot ID
1654	* @param string $content_type The content type
1655	* @param MxChat_Chunker $chunker The chunker instance
1656	* @return bool\|WP_Error True on success, WP_Error on failure
1657	*/
1658	private static function submit_chunked_content($content, $source_url, $api_key, $bot_id, $content_type, $chunker) {
1659	global $wpdb;
1660	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1661
1662	//error_log('[MXCHAT-CHUNK-DEBUG] Starting chunked submission for: ' . $source_url);
1663	//error_log('[MXCHAT-CHUNK-DEBUG] Content length: ' . strlen($content) . ' chars');
1664
1665	// URL-less (manual) content needs a minted identity BEFORE chunk ids are derived:
1666	// every chunk id is md5(source_url)_chunk_N, so with source_url = '' EVERY long
1667	// manual document shared the md5('') prefix — and the clean-slate delete below
1668	// wiped the PREVIOUS manual entry's chunks each time a new one was added. The
1669	// single-vector paths already mint (mxchat:// in WP, manual_* in Pinecone); this
1670	// was the one storage path that didn't. Keep an mxchat:// identity if the caller
1671	// already carries one.
1672	if (strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, 'upload://') !== 0 && !preg_match('#^https?://#i', $source_url)) {
1673	$source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
1674	}
1675
1676	// First, delete any existing chunks for this URL (clean slate).
1677	// Mirror-suspended: this is a re-store, not an entry removal — the Vector
1678	// Store file is replaced (or kept, on hash match) by the caller's
1679	// sync_upsert_entry after storage succeeds.
1680	self::$vectorstore_mirror_suspended = true;
1681	$delete_result = self::delete_chunks_for_url($source_url, $bot_id);
1682	self::$vectorstore_mirror_suspended = false;
1683	if (is_wp_error($delete_result)) {
1684	//error_log('[MXCHAT-CHUNK-DEBUG] Warning: Failed to delete existing chunks: ' . $delete_result->get_error_message());
1685	// Continue anyway - we'll overwrite with upsert
1686	}
1687
1688	// Split content into chunks
1689	$chunks = $chunker->chunk_text($content);
1690	$total_chunks = count($chunks);
1691
1692	//error_log('[MXCHAT-CHUNK-DEBUG] Created ' . $total_chunks . ' chunks');
1693	foreach ($chunks as $i => $chunk) {
1694	//error_log('[MXCHAT-CHUNK-DEBUG] Chunk ' . $i . ' length: ' . strlen($chunk) . ' chars, preview: ' . substr($chunk, 0, 100));
1695	}
1696
1697	//error_log('[MXCHAT-CHUNK] Split content into ' . $total_chunks . ' chunks');
1698
1699	if ($total_chunks === 0) {
1700	return new WP_Error('chunking_failed', 'Content could not be split into chunks');
1701	}
1702
1703	$errors = array();
1704	$embed_failures = 0;
1705	$first_embed_reason = '';
1706	$first_store_reason = '';
1707	$is_pinecone = self::is_pinecone_enabled_for_bot($bot_id);
1708
1709	foreach ($chunks as $index => $chunk_text) {
1710	// Generate chunk metadata
1711	$chunk_metadata = MxChat_Chunker::create_chunk_metadata($index, $total_chunks, $source_url);
1712
1713	// AI-Engine-style aliases so external consumers (Pinecone/Qdrant/Chroma) can rely on
1714	// a stable shorthand ('source'/'part_index'/'part_total') without parsing our internal names.
1715	$chunk_metadata['source'] = $source_url;
1716	$chunk_metadata['part_index'] = (int) $index;
1717	$chunk_metadata['part_total'] = (int) $total_chunks;
1718
1719	/**
1720	* Filter the per-chunk metadata blob before it's written to the KB store.
1721	*
1722	* @param array $chunk_metadata Metadata array (source, part_index, part_total, chunk_index, total_chunks, source_url, parent_url_hash, document_type, ...).
1723	* @param string $chunk_text The chunk text being stored.
1724	* @param array $context ['bot_id' => string, 'content_type' => string, 'source_url' => string, 'part_index' => int, 'part_total' => int]
1725	* @return array Updated metadata array.
1726	*/
1727	$chunk_metadata = apply_filters(
1728	'mxchat_embedding_chunk_metadata',
1729	$chunk_metadata,
1730	$chunk_text,
1731	array(
1732	'bot_id' => $bot_id,
1733	'content_type' => $content_type,
1734	'source_url' => $source_url,
1735	'part_index' => (int) $index,
1736	'part_total' => (int) $total_chunks,
1737	)
1738	);
1739
1740	$chunk_vector_id = MxChat_Chunker::generate_chunk_vector_id($source_url, $index);
1741
1742	//error_log('[MXCHAT-CHUNK] Processing chunk ' . ($index + 1) . '/' . $total_chunks . ' (ID: ' . $chunk_vector_id . ')');
1743
1744	// Generate embedding for this chunk
1745	$embedding_vector = self::generate_embedding($chunk_text, $api_key, $bot_id);
1746
1747	if (!is_array($embedding_vector)) {
1748	// Track embedding failures separately from storage failures, and
1749	// keep the first provider reason seen — the two failure classes
1750	// have opposite remedies (API key vs Pinecone/DB) (plan 4a7c0a).
1751	$embed_failures++;
1752	$reason = is_wp_error($embedding_vector) ? $embedding_vector->get_error_message() : '';
1753	if ($reason !== '' && $first_embed_reason === '') {
1754	$first_embed_reason = $reason;
1755	}
1756	$errors[] = new WP_Error('embedding_failed', 'Failed to generate embedding for chunk ' . $index . ($reason !== '' ? ' — ' . $reason : ''));
1757	continue;
1758	}
1759
1760	if ($is_pinecone) {
1761	// Store in Pinecone with chunk metadata
1762	$result = self::store_chunk_in_pinecone(
1763	$embedding_vector,
1764	$chunk_text,
1765	$source_url,
1766	$chunk_vector_id,
1767	$bot_id,
1768	$content_type,
1769	$chunk_metadata
1770	);
1771	} else {
1772	// Store in WordPress DB with chunk metadata
1773	$content_with_metadata = MxChat_Chunker::format_chunk_for_storage($chunk_text, $chunk_metadata);
1774	$embedding_vector_serialized = maybe_serialize($embedding_vector);
1775
1776	$result = self::store_chunk_in_wordpress_db(
1777	$content_with_metadata,
1778	$source_url,
1779	$embedding_vector_serialized,
1780	$table_name,
1781	$content_type,
1782	$chunk_metadata
1783	);
1784	}
1785
1786	if (is_wp_error($result)) {
1787	$errors[] = $result;
1788	if ($first_store_reason === '') {
1789	$first_store_reason = $result->get_error_message();
1790	}
1791	}
1792	}
1793
1794	if (count($errors) === $total_chunks) {
1795	// Say WHICH stage failed — "failed to store" used to cover pure
1796	// embedding failures too, sending customers to debug Pinecone when
1797	// the problem was their embedding API key (plan 4a7c0a).
1798	if ($embed_failures === $total_chunks) {
1799	return new WP_Error('chunking_failed',
1800	'Failed to store any chunks — every chunk failed to embed'
1801	. ($first_embed_reason !== '' ? ': ' . $first_embed_reason : '')
1802	. ' Check the embedding provider API key and model under MxChat Settings.');
1803	}
1804	if ($embed_failures === 0) {
1805	return new WP_Error('chunking_failed',
1806	'Failed to store any chunks — embeddings generated but storage failed'
1807	. ($first_store_reason !== '' ? ': ' . $first_store_reason : '')
1808	. ' Check the knowledge base storage (Pinecone index or database).');
1809	}
1810	return new WP_Error('chunking_failed', sprintf(
1811	'Failed to store any chunks — %d failed to embed%s and %d failed to store%s',
1812	$embed_failures,
1813	$first_embed_reason !== '' ? ' (' . $first_embed_reason . ')' : '',
1814	$total_chunks - $embed_failures,
1815	$first_store_reason !== '' ? ' (' . $first_store_reason . ')' : ''
1816	));
1817	}
1818
1819	if (!empty($errors)) {
1820	$detail = $first_embed_reason !== '' ? $first_embed_reason : $first_store_reason;
1821	return new WP_Error('chunking_partial_failure',
1822	sprintf('Failed to store %d of %d chunks', count($errors), $total_chunks)
1823	. ($detail !== '' ? ' — first error: ' . $detail : ''));
1824	}
1825
1826	//error_log('[MXCHAT-CHUNK] Successfully stored all ' . $total_chunks . ' chunks');
1827	return true;
1828	}
1829
1830	/**
1831	* Store a single chunk in Pinecone with chunk-specific metadata
1832	*/
1833	private static function store_chunk_in_pinecone($embedding_vector, $chunk_text, $source_url, $vector_id, $bot_id, $content_type, $chunk_metadata) {
1834	// Get Pinecone configuration
1835	if ($bot_id === 'default' \|\| !class_exists('MxChat_Multi_Bot_Manager')) {
1836	$pinecone_options = get_option('mxchat_pinecone_addon_options');
1837	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1838	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1839	$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1840	} else {
1841	$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1842	if (empty($bot_pinecone_config)) {
1843	$pinecone_options = get_option('mxchat_pinecone_addon_options');
1844	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1845	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
1846	$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
1847	} else {
1848	$api_key = $bot_pinecone_config['api_key'] ?? '';
1849	$host = $bot_pinecone_config['host'] ?? '';
1850	$namespace = $bot_pinecone_config['namespace'] ?? '';
1851	}
1852	}
1853
1854	if (empty($host) \|\| empty($api_key)) {
1855	return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
1856	}
1857
1858	$api_endpoint = "https://{$host}/vectors/upsert";
1859
1860	// Build metadata with chunk information
1861	$metadata = array(
1862	'text' => $chunk_text,
1863	'source_url' => $source_url,
1864	'type' => $content_type,
1865	'is_chunked' => true,
1866	'chunk_index' => $chunk_metadata['chunk_index'],
1867	'total_chunks' => $chunk_metadata['total_chunks'],
1868	'parent_url_hash' => $chunk_metadata['parent_url_hash'],
1869	'last_updated' => time(),
1870	'created_at' => time(),
1871	'bot_id' => $bot_id,
1872	);
1873	$page_title = self::source_title_for($source_url);
1874	if ($page_title !== '') {
1875	$metadata['title'] = $page_title; // page title captured at import (plan d9ee66)
1876	}
1877
1878	// Document index (plan 362c31): same chunk id and metadata, documents API.
1879	if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) {
1880	return MxChat_Pinecone_Documents::upsert_one($vector_id, $chunk_text, $embedding_vector, $metadata, array(
1881	'api_key' => $api_key,
1882	'host' => $host,
1883	'namespace' => $namespace,
1884	));
1885	}
1886
1887	$vector_data = array(
1888	'id' => $vector_id,
1889	'values' => $embedding_vector,
1890	'metadata' => $metadata
1891	);
1892
1893	$request_body = array(
1894	'vectors' => array($vector_data)
1895	);
1896
1897	if (!empty($namespace)) {
1898	$request_body['namespace'] = $namespace;
1899	}
1900
1901	$response = wp_remote_post($api_endpoint, array(
1902	'headers' => array(
1903	'Api-Key' => $api_key,
1904	'accept' => 'application/json',
1905	'content-type' => 'application/json'
1906	),
1907	'body' => wp_json_encode($request_body),
1908	'timeout' => 30
1909	));
1910
1911	if (is_wp_error($response)) {
1912	return $response;
1913	}
1914
1915	$response_code = wp_remote_retrieve_response_code($response);
1916	if ($response_code !== 200) {
1917	return new WP_Error('pinecone_api', 'Pinecone API error: HTTP ' . $response_code);
1918	}
1919
1920	return true;
1921	}
1922
1923	/**
1924	* Store a single chunk in WordPress database
1925	*/
1926	private static function store_chunk_in_wordpress_db($content_with_metadata, $source_url, $embedding_vector_serialized, $table_name, $content_type, $chunk_metadata) {
1927	global $wpdb;
1928
1929	// For chunks, we always insert new rows (no duplicate checking)
1930	// The URL includes chunk info in the metadata, but source_url stays the same for grouping
1931	$row = array(
1932	'url' => $source_url,
1933	'article_content' => $content_with_metadata,
1934	'embedding_vector' => $embedding_vector_serialized,
1935	'source_url' => $source_url,
1936	'content_type' => $content_type,
1937	'timestamp' => current_time('mysql')
1938	);
1939	$formats = array('%s', '%s', '%s', '%s', '%s', '%s');
1940	// Every chunk row carries the page title too (plan d9ee66) - readers group by
1941	// source_url and may pick any chunk as the representative.
1942	$page_title = self::source_title_for($source_url);
1943	if ($page_title !== '' && self::ensure_kb_title_column()) {
1944	$row['title'] = $page_title;
1945	$formats[] = '%s';
1946	}
1947	$result = $wpdb->insert($table_name, $row, $formats);
1948
1949	if ($result === false) {
1950	return new WP_Error('database_failed', 'Failed to insert chunk: ' . $wpdb->last_error);
1951	}
1952
1953	return true;
1954	}
1955
1956	/**
1957	* Option holding the page an uploaded file should be cited as (plan 025ea2).
1958	* Keyed "<bot_id>\|upload://<filename>" => https URL. The entry itself keeps
1959	* its upload:// identity, so a re-upload of the same file still replaces the
1960	* same entry and never collides with the cited page's own knowledge entry.
1961	*/
1962	const UPLOAD_SOURCE_URLS_OPTION = 'mxchat_upload_source_urls';
1963
1964	/**
1965	* Whether a stored source is an uploaded file (Document Upload / PDF Upload).
1966	*
1967	* @param string $source_url Stored source identity.
1968	* @return bool
1969	*/
1970	public static function is_upload_source($source_url) {
1971	return is_string($source_url) && stripos($source_url, 'upload://') === 0;
1972	}
1973
1974	/**
1975	* File name of an uploaded source, for a plain-text label.
1976	*
1977	* @param string $source_url Stored source identity (upload://<filename>).
1978	* @return string
1979	*/
1980	public static function upload_source_name($source_url) {
1981	return self::is_upload_source($source_url) ? substr(self::upload_source_base($source_url), strlen('upload://')) : '';
1982	}
1983
1984	/**
1985	* An uploaded source without its page fragment (plan dcc19d). A page of an
1986	* uploaded PDF is stored as upload://<filename>#page=<n>; the file it belongs
1987	* to, and the key its Source URL is kept under, is upload://<filename>.
1988	*
1989	* @param string $source_url Stored source identity.
1990	* @return string
1991	*/
1992	public static function upload_source_base($source_url) {
1993	return is_string($source_url) ? preg_replace('/#page=\d+$/', '', $source_url) : '';
1994	}
1995
1996	/**
1997	* Page number carried by an uploaded PDF page's identity, 0 when it has none.
1998	*
1999	* @param string $source_url Stored source identity.
2000	* @return int
2001	*/
2002	public static function upload_source_page($source_url) {
2003	return (self::is_upload_source($source_url) && preg_match('/#page=(\d+)$/', $source_url, $m)) ? (int) $m[1] : 0;
2004	}
2005
2006	/**
2007	* Identity one page of a PDF is stored under: <pdf>#page=<n>. A web PDF goes
2008	* through esc_url() as before. An uploaded PDF (upload://<filename>) must not:
2009	* esc_url() empties that scheme, which stored every page of an uploaded PDF
2010	* as anonymous manual content, so the file could not be named, cited, given
2011	* a Source URL, or replaced by a re-upload (plan dcc19d).
2012	*
2013	* @param string $pdf_url The PDF's own source (URL or upload://<filename>).
2014	* @param int $page_number 1-based page number.
2015	* @return string
2016	*/
2017	public static function pdf_page_identity($pdf_url, $page_number) {
2018	$identity = $pdf_url . '#page=' . (int) $page_number;
2019	return self::is_upload_source($pdf_url) ? sanitize_text_field($identity) : esc_url($identity);
2020	}
2021
2022	/**
2023	* Remember (or clear) the page an uploaded file should be cited as.
2024	* Only http(s) URLs are kept; anything else clears the entry.
2025	*
2026	* @param string $source_label upload://<filename>
2027	* @param string $bot_id Bot the upload belongs to.
2028	* @param string $url Page URL, or '' to clear.
2029	* @return string The URL stored, '' when cleared.
2030	*/
2031	public static function set_upload_source_url($source_label, $bot_id, $url) {
2032	if (!self::is_upload_source($source_label)) {
2033	return '';
2034	}
2035	$bot_id = ($bot_id === '' \|\| $bot_id === null) ? 'default' : (string) $bot_id;
2036	$url = is_string($url) ? esc_url_raw(trim($url), array('http', 'https')) : '';
2037	if ($url !== '' && !preg_match('#^https?://#i', $url)) {
2038	$url = '';
2039	}
2040
2041	$map = get_option(self::UPLOAD_SOURCE_URLS_OPTION, array());
2042	if (!is_array($map)) {
2043	$map = array();
2044	}
2045	$key = $bot_id . '\|' . $source_label;
2046	if ($url === '') {
2047	if (!isset($map[$key])) {
2048	return '';
2049	}
2050	unset($map[$key]);
2051	} else {
2052	$map[$key] = $url;
2053	}
2054	if (empty($map)) {
2055	delete_option(self::UPLOAD_SOURCE_URLS_OPTION);
2056	} else {
2057	update_option(self::UPLOAD_SOURCE_URLS_OPTION, $map, false);
2058	}
2059	return $url;
2060	}
2061
2062	/**
2063	* The page an uploaded file is cited as, '' when it has none.
2064	* Looks under the asking bot first, then under the default bot (on the
2065	* WordPress database store every bot reads the same entries).
2066	*
2067	* @param string $source_label upload://<filename>
2068	* @param string $bot_id Bot answering the turn.
2069	* @return string
2070	*/
2071	public static function get_upload_source_url($source_label, $bot_id = 'default') {
2072	if (!self::is_upload_source($source_label)) {
2073	return '';
2074	}
2075	$map = get_option(self::UPLOAD_SOURCE_URLS_OPTION, array());
2076	if (!is_array($map) \|\| empty($map)) {
2077	return '';
2078	}
2079	// A PDF page (upload://<file>#page=<n>) is cited as its file's page.
2080	$source_label = self::upload_source_base($source_label);
2081	$bot_id = ($bot_id === '' \|\| $bot_id === null) ? 'default' : (string) $bot_id;
2082	foreach (array($bot_id . '\|' . $source_label, 'default\|' . $source_label) as $key) {
2083	if (!empty($map[$key]) && preg_match('#^https?://#i', $map[$key])) {
2084	return $map[$key];
2085	}
2086	}
2087	return '';
2088	}
2089
2090	/**
2091	* Delete all chunks for a given URL
2092	*
2093	* @param string $source_url The source URL
2094	* @param string $bot_id The bot ID
2095	* @return bool\|WP_Error True on success, WP_Error on failure
2096	*/
2097	public static function delete_chunks_for_url($source_url, $bot_id = 'default') {
2098	//error_log('[MXCHAT-CHUNK-DELETE] Deleting chunks for URL: ' . $source_url);
2099
2100	// Entry removal — mirror it to the Vector Store (unless a storage routine
2101	// is mid-re-store, see $vectorstore_mirror_suspended).
2102	if (!self::$vectorstore_mirror_suspended && class_exists('MxChat_Vectorstore_Manager')) {
2103	MxChat_Vectorstore_Manager::sync_delete_entry($source_url, $bot_id);
2104	}
2105
2106	if (self::is_pinecone_enabled_for_bot($bot_id)) {
2107	$result = self::delete_pinecone_chunks_by_url($source_url, $bot_id);
2108	} else {
2109	$result = self::delete_wordpress_chunks_by_url($source_url);
2110	}
2111
2112	// The entry is gone for good (not the clean-slate half of a re-store):
2113	// a media file's "already indexed, nothing changed" signature must go
2114	// with it, or auto-sync keeps skipping the file on every later save.
2115	if (!self::$vectorstore_mirror_suspended && !is_wp_error($result)) {
2116	self::forget_attachment_signature($source_url, $bot_id);
2117	}
2118
2119	return $result;
2120	}
2121
2122	/**
2123	* Clear the media auto-sync signature (_mxchat_kb_attachment_sig) of the
2124	* attachment a just-deleted knowledge entry was indexed from.
2125	*
2126	* Call it wherever a WHOLE entry is removed (never for the clean-slate half
2127	* of a re-store, and never when sibling chunk rows survive). Posts and pages
2128	* carry no signature; for them this is two indexed lookups that find nothing.
2129	*
2130	* @param string $source_url The entry's source URL.
2131	* @param string $bot_id The bot whose entry was removed.
2132	*/
2133	public static function forget_attachment_signature($source_url, $bot_id = 'default') {
2134	global $wpdb;
2135
2136	$source_url = (string) $source_url;
2137	if (!preg_match('#^https?://#i', $source_url) \|\| !self::attachment_signature_follows_bot($bot_id)) {
2138	return;
2139	}
2140
2141	$attachment_ids = array();
2142	$by_url = attachment_url_to_postid($source_url);
2143	if ($by_url) {
2144	$attachment_ids[] = (int) $by_url;
2145	}
2146
2147	// The signature stores the exact URL the indexer used, which can differ
2148	// from where the file lives now (replaced file, scaled variant). Bounded
2149	// by the number of signed attachments: the meta_key index does the work.
2150	$candidates = $wpdb->get_results($wpdb->prepare(
2151	"SELECT post_id, meta_value FROM {$wpdb->postmeta}
2152	WHERE meta_key = '_mxchat_kb_attachment_sig' AND meta_value LIKE %s
2153	LIMIT 20",
2154	'%' . $wpdb->esc_like($source_url) . '%'
2155	));
2156	foreach ((array) $candidates as $candidate) {
2157	$signature = maybe_unserialize($candidate->meta_value);
2158	if (is_array($signature) && isset($signature['url']) && $signature['url'] === $source_url) {
2159	$attachment_ids[] = (int) $candidate->post_id;
2160	}
2161	}
2162
2163	foreach (array_unique($attachment_ids) as $attachment_id) {
2164	delete_post_meta($attachment_id, '_mxchat_kb_attachment_sig');
2165	}
2166	}
2167
2168	/**
2169	* The same, for callers that hold a Pinecone vector id instead of a URL.
2170	* Only a base id (md5 of the source URL) names a whole entry; a single
2171	* chunk id leaves the rest of the entry in place, so it is ignored.
2172	*/
2173	public static function forget_attachment_signature_by_key($entry_key, $bot_id = 'default') {
2174	global $wpdb;
2175
2176	$entry_key = (string) $entry_key;
2177	if (!preg_match('/^[a-f0-9]{32}$/', $entry_key) \|\| !self::attachment_signature_follows_bot($bot_id)) {
2178	return;
2179	}
2180
2181	$signed = $wpdb->get_results(
2182	"SELECT post_id, meta_value FROM {$wpdb->postmeta}
2183	WHERE meta_key = '_mxchat_kb_attachment_sig' LIMIT 5000"
2184	);
2185	foreach ((array) $signed as $row) {
2186	$signature = maybe_unserialize($row->meta_value);
2187	if (is_array($signature) && isset($signature['url']) && md5((string) $signature['url']) === $entry_key) {
2188	delete_post_meta((int) $row->post_id, '_mxchat_kb_attachment_sig');
2189	}
2190	}
2191	}
2192
2193	/**
2194	* Every media entry of a bot was removed at once ("Delete all").
2195	*/
2196	public static function forget_all_attachment_signatures($bot_id = 'default') {
2197	if ($bot_id !== 'default' && $bot_id !== 'testing' && !self::is_pinecone_enabled_for_bot($bot_id)) {
2198	// WordPress-database "Delete all" for another bot removes that bot's
2199	// rows only; the default bot's entries are still there.
2200	return;
2201	}
2202	if (self::attachment_signature_follows_bot($bot_id)) {
2203	delete_post_meta_by_key('_mxchat_kb_attachment_sig');
2204	}
2205	}
2206
2207	/**
2208	* The signature vouches for the DEFAULT bot's entry (media auto-sync only
2209	* indexes for the default bot). Removing another bot's copy from a Pinecone
2210	* store of its own leaves that entry, and so the signature, in place.
2211	*/
2212	private static function attachment_signature_follows_bot($bot_id) {
2213	if ($bot_id === 'default' \|\| $bot_id === 'testing' \|\| !self::is_pinecone_enabled_for_bot($bot_id)) {
2214	return true;
2215	}
2216	if (!class_exists('MxChat_Pinecone_Documents') \|\| !self::is_pinecone_enabled_for_bot('default')) {
2217	return false;
2218	}
2219	$theirs = MxChat_Pinecone_Documents::config($bot_id);
2220	$ours = MxChat_Pinecone_Documents::config('default');
2221	return strtolower((string) $theirs['host']) === strtolower((string) $ours['host'])
2222	&& (string) $theirs['namespace'] === (string) $ours['namespace'];
2223	}
2224
2225	/**
2226	* Delete all chunks for a URL from Pinecone
2227	*/
2228	/**
2229	* Delete leftover md5(url)_chunk_N vectors after a URL's content was re-stored
2230	* as a SINGLE vector (content shrank below the chunk threshold on edit/re-import).
2231	* Unlike delete_pinecone_chunks_by_url this leaves the base id alone — the caller
2232	* just upserted the new content there. Failure is logged, not fatal: the save
2233	* itself succeeded, and the next save retries the sweep.
2234	*/
2235	private static function cleanup_pinecone_chunk_stragglers($source_url, $bot_id) {
2236	if ($bot_id === 'default' \|\| !class_exists('MxChat_Multi_Bot_Manager')) {
2237	$pinecone_options = get_option('mxchat_pinecone_addon_options');
2238	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2239	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
2240	$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
2241	} else {
2242	$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
2243	if (empty($bot_pinecone_config)) {
2244	$pinecone_options = get_option('mxchat_pinecone_addon_options');
2245	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
2246	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
2247	$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
2248	} else {
2249	$api_key = $bot_pinecone_config['api_key'] ?? '';
2250	$host = $bot_pinecone_config['host'] ?? '';
2251	$namespace = $bot_pinecone_config['namespace'] ?? '';
2252	}
2253	}
2254
2255	if (empty($host) \|\| empty($api_key)) {
2256	return;
2257	}
2258
2259	// Document index (plan 362c31): list the chunk ids and delete them through the documents API.
2260	if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) {
2261	$docs_cfg = array('api_key' => $api_key, 'host' => $host, 'namespace' => $namespace);
2262	$doc_stragglers = MxChat_Pinecone_Documents::list_ids(md5($source_url) . '_chunk_', $docs_cfg);
2263	if (!empty($doc_stragglers)) {
2264	$doc_result = MxChat_Pinecone_Documents::delete_ids($doc_stragglers, $docs_cfg);
2265	if (is_wp_error($doc_result) && class_exists('MxChat_Admin') && method_exists('MxChat_Admin', 'mxchat_log_debug')) {
2266	MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to sweep stale chunk documents after single-vector re-store', array('source_url' => $source_url, 'bot_id' => $bot_id, 'count' => count($doc_stragglers)));
2267	}
2268	}
2269	return;
2270	}
2271
2272	$stragglers = array();
2273
2274	// Pinecone /vectors/list is a GET endpoint with query-string parameters (a POST
2275	// answers 200-with-an-empty-body, which reads as "no stragglers").
2276	$query_params = array(
2277	'prefix' => md5($source_url) . '_chunk_',
2278	'limit' => 100,
2279	);
2280	if (!empty($namespace)) {
2281	$query_params['namespace'] = $namespace;
2282	}
2283
2284	$list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
2285
2286	do {
2287	$list_response = wp_remote_get($list_url, array(
2288	'headers' => array(
2289	'Api-Key' => $api_key,
2290	'accept' => 'application/json',
2291	),
2292	'timeout' => 30,
2293	));
2294
2295	if (is_wp_error($list_response) \|\| wp_remote_retrieve_response_code($list_response) !== 200) {
2296	break;
2297	}
2298
2299	$list_data = json_decode(wp_remote_retrieve_body($list_response), true);
2300	if (!empty($list_data['vectors'])) {
2301	foreach ($list_data['vectors'] as $vector) {
2302	if (isset($vector['id'])) {
2303	$stragglers[] = $vector['id'];
2304	}
2305	}
2306	}
2307
2308	$next_token = $list_data['pagination']['next'] ?? '';
2309	if (empty($next_token)) {
2310	break;
2311	}
2312
2313	$query_params['paginationToken'] = $next_token;
2314	$list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
2315	} while (true);
2316
2317	if (empty($stragglers)) {
2318	return;
2319	}
2320
2321	$delete_body = array('ids' => $stragglers);
2322	if (!empty($namespace)) {
2323	$delete_body['namespace'] = $namespace;
2324	}
2325
2326	$delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
2327	'headers' => array(
2328	'Api-Key' => $api_key,
2329	'accept' => 'application/json',
2330	'content-type' => 'application/json'
2331	),
2332	'body' => wp_json_encode($delete_body),
2333	'timeout' => 30
2334	));
2335
2336	if ((is_wp_error($delete_response) \|\| wp_remote_retrieve_response_code($delete_response) !== 200)
2337	&& class_exists('MxChat_Admin') && method_exists('MxChat_Admin', 'mxchat_log_debug')) {
2338	MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to sweep stale chunk vectors after single-vector re-store', array('source_url' => $source_url, 'bot_id' => $bot_id, 'count' => count($stragglers)));
2339	}
2340	}
2341
2342	/**
2343	* Resolve the Pinecone credentials for a bot: the bot's own config when the
2344	* multi-bot add-on supplies one, else the site-wide options. Extracted so the
2345	* delete path and the count path can never resolve different credentials.
2346	*
2347	* @param string $bot_id
2348	* @return array{api_key:string,host:string,namespace:string}
2349	*/
2350	private static function pinecone_config_for_bot($bot_id) {
2351	if ($bot_id === 'default' \|\| !class_exists('MxChat_Multi_Bot_Manager')) {
2352	$bot_pinecone_config = array();
2353	} else {
2354	$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
2355	}
2356
2357	if (empty($bot_pinecone_config)) {
2358	$pinecone_options = get_option('mxchat_pinecone_addon_options');
2359	return array(
2360	'api_key' => $pinecone_options['mxchat_pinecone_api_key'] ?? '',
2361	'host' => $pinecone_options['mxchat_pinecone_host'] ?? '',
2362	'namespace' => $pinecone_options['mxchat_pinecone_namespace'] ?? '',
2363	// 'vector' \| 'document' (plan 362c31) — the delete / count paths branch on it.
2364	'index_type' => (($pinecone_options['mxchat_pinecone_index_type'] ?? 'vector') === 'document') ? 'document' : 'vector',
2365	);
2366	}
2367
2368	return array(
2369	'api_key' => $bot_pinecone_config['api_key'] ?? '',
2370	'host' => $bot_pinecone_config['host'] ?? '',
2371	'namespace' => $bot_pinecone_config['namespace'] ?? '',
2372	'index_type' => (($bot_pinecone_config['index_type'] ?? 'vector') === 'document') ? 'document' : 'vector',
2373	);
2374	}
2375
2376	/**
2377	* List every Pinecone vector id under a prefix, following pagination.
2378	*
2379	* Pinecone /vectors/list is a GET endpoint with query-string parameters; a POST here returns a
2380	* non-200 silently and we end up seeing only the base vector, leaving chunks orphaned.
2381	*
2382	* @param string $prefix Vector-id prefix to match.
2383	* @param array $cfg From pinecone_config_for_bot().
2384	* @return array List of vector ids (empty when none match or the API is unreachable).
2385	*/
2386	private static function pinecone_list_ids_by_prefix($prefix, array $cfg) {
2387	// Document index (plan 362c31): the documents API lists by prefix too.
2388	if (($cfg['index_type'] ?? 'vector') === 'document' && class_exists('MxChat_Pinecone_Documents')) {
2389	return MxChat_Pinecone_Documents::list_ids($prefix, $cfg);
2390	}
2391
2392	$ids = array();
2393
2394	$query_params = array(
2395	'prefix' => $prefix,
2396	'limit' => 100,
2397	);
2398	if (!empty($cfg['namespace'])) {
2399	$query_params['namespace'] = $cfg['namespace'];
2400	}
2401
2402	$list_url = "https://{$cfg['host']}/vectors/list?" . http_build_query($query_params);
2403
2404	// Paginate in case a URL has more than 100 chunks.
2405	do {
2406	$list_response = wp_remote_get($list_url, array(
2407	'headers' => array(
2408	'Api-Key' => $cfg['api_key'],
2409	'accept' => 'application/json',
2410	),
2411	'timeout' => 30,
2412	));
2413
2414	if (is_wp_error($list_response) \|\| wp_remote_retrieve_response_code($list_response) !== 200) {
2415	break;
2416	}
2417
2418	$list_data = json_decode(wp_remote_retrieve_body($list_response), true);
2419	if (!empty($list_data['vectors'])) {
2420	foreach ($list_data['vectors'] as $vector) {
2421	if (isset($vector['id'])) {
2422	$ids[] = $vector['id'];
2423	}
2424	}
2425	}
2426
2427	$next_token = $list_data['pagination']['next'] ?? '';
2428	if (empty($next_token)) {
2429	break;
2430	}
2431
2432	$query_params['paginationToken'] = $next_token;
2433	$list_url = "https://{$cfg['host']}/vectors/list?" . http_build_query($query_params);
2434	} while (true);
2435
2436	return $ids;
2437	}
2438
2439	/**
2440	* How many stored pieces a URL currently has, in whichever backend holds it.
2441	*
2442	* Branches on the SAME condition delete_chunks_for_url() branches on, so the
2443	* number reported is the number that call removes. Used by the REST DELETE
2444	* route to answer "was anything actually there?" without a second delete path.
2445	*
2446	* @param string $source_url
2447	* @param string $bot_id
2448	* @return int
2449	*/
2450	public static function count_chunks_for_url($source_url, $bot_id = 'default') {
2451	if (self::is_pinecone_enabled_for_bot($bot_id)) {
2452	$cfg = self::pinecone_config_for_bot($bot_id);
2453	if (empty($cfg['host']) \|\| empty($cfg['api_key'])) {
2454	return 0;
2455	}
2456	// The base id and every "<base>_chunk_N" share the base as a prefix,
2457	// so one prefix listing covers chunked and non-chunked content alike.
2458	return count(self::pinecone_list_ids_by_prefix(md5($source_url), $cfg));
2459	}
2460
2461	global $wpdb;
2462	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2463
2464	return (int) $wpdb->get_var($wpdb->prepare(
2465	"SELECT COUNT(*) FROM {$table_name} WHERE source_url = %s",
2466	$source_url
2467	));
2468	}
2469
2470	private static function delete_pinecone_chunks_by_url($source_url, $bot_id) {
2471	$cfg = self::pinecone_config_for_bot($bot_id);
2472	$api_key = $cfg['api_key'];
2473	$host = $cfg['host'];
2474	$namespace = $cfg['namespace'];
2475
2476	if (empty($host) \|\| empty($api_key)) {
2477	return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
2478	}
2479
2480	// Document index (plan 362c31): base id + every <md5>_chunk_N, documents API.
2481	if (($cfg['index_type'] ?? 'vector') === 'document' && class_exists('MxChat_Pinecone_Documents')) {
2482	$doc_ids = array_merge(array(md5($source_url)), MxChat_Pinecone_Documents::list_ids(md5($source_url) . '_chunk_', $cfg));
2483	return MxChat_Pinecone_Documents::delete_ids($doc_ids, $cfg);
2484	}
2485
2486	$base_vector_id = md5($source_url);
2487	$vectors_to_delete = array();
2488
2489	// Add the original single-vector ID (for non-chunked content)
2490	$vectors_to_delete[] = $base_vector_id;
2491
2492	$vectors_to_delete = array_merge(
2493	$vectors_to_delete,
2494	self::pinecone_list_ids_by_prefix($base_vector_id . '_chunk_', $cfg)
2495	);
2496
2497	if (empty($vectors_to_delete)) {
2498	//error_log('[MXCHAT-CHUNK-DELETE] No vectors found to delete');
2499	return true;
2500	}
2501
2502	//error_log('[MXCHAT-CHUNK-DELETE] Deleting ' . count($vectors_to_delete) . ' vectors from Pinecone');
2503
2504	// Delete vectors
2505	$delete_url = "https://{$host}/vectors/delete";
2506
2507	$delete_body = array(
2508	'ids' => $vectors_to_delete
2509	);
2510
2511	if (!empty($namespace)) {
2512	$delete_body['namespace'] = $namespace;
2513	}
2514
2515	$delete_response = wp_remote_post($delete_url, array(
2516	'headers' => array(
2517	'Api-Key' => $api_key,
2518	'accept' => 'application/json',
2519	'content-type' => 'application/json'
2520	),
2521	'body' => wp_json_encode($delete_body),
2522	'timeout' => 30
2523	));
2524
2525	if (is_wp_error($delete_response)) {
2526	return $delete_response;
2527	}
2528
2529	$response_code = wp_remote_retrieve_response_code($delete_response);
2530	if ($response_code !== 200) {
2531	return new WP_Error('pinecone_delete', 'Failed to delete vectors: HTTP ' . $response_code);
2532	}
2533
2534	return true;
2535	}
2536
2537	/**
2538	* Delete all chunks for a URL from WordPress database
2539	*/
2540	private static function delete_wordpress_chunks_by_url($source_url) {
2541	global $wpdb;
2542	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
2543
2544	// Delete all rows with this source_url (handles both chunked and non-chunked)
2545	$result = $wpdb->delete(
2546	$table_name,
2547	array('source_url' => $source_url),
2548	array('%s')
2549	);
2550
2551	if ($result === false) {
2552	return new WP_Error('database_delete', 'Failed to delete chunks: ' . $wpdb->last_error);
2553	}
2554
2555	//error_log('[MXCHAT-CHUNK-DELETE] Deleted ' . $result . ' rows from WordPress DB');
2556	return true;
2557	}
2558
2559	/**
2560	* Hybrid keyword boost (plan-38ffa1): detect whether the WP-DB knowledge
2561	* table can serve the keyword leg via a MySQL FULLTEXT index, creating the
2562	* index if needed. Detection runs once and caches the answer in the
2563	* mxchat_hybrid_keyword_capability option ('fulltext' \| 'like'); pass
2564	* $force to re-detect. LIKE is the graceful fallback for shared hosts
2565	* whose ALTER fails — the feature works either way, FULLTEXT just ranks
2566	* better and scales.
2567	*
2568	* @param bool $force Re-run detection even if a cached answer exists.
2569	* @return string 'fulltext' or 'like'
2570	*/
2571	public static function mxchat_hybrid_detect_capability($force = false) {
2572	$cached = get_option('mxchat_hybrid_keyword_capability', '');
2573	if (!$force && in_array($cached, array('fulltext', 'like'), true)) {
2574	return $cached;
2575	}
2576
2577	global $wpdb;
2578	$table = $wpdb->prefix . 'mxchat_system_prompt_content';
2579
2580	$index_exists = $wpdb->get_var("SHOW INDEX FROM {$table} WHERE Key_name = 'mxchat_content_ft'");
2581	if (!$index_exists) {
2582	// Suppress the visible error on hosts where this is not permitted —
2583	// failure is an expected, handled outcome (LIKE fallback).
2584	$suppress = $wpdb->suppress_errors(true);
2585	$wpdb->query("ALTER TABLE {$table} ADD FULLTEXT INDEX mxchat_content_ft (article_content)");
2586	$wpdb->suppress_errors($suppress);
2587	$index_exists = $wpdb->get_var("SHOW INDEX FROM {$table} WHERE Key_name = 'mxchat_content_ft'");
2588	}
2589
2590	$capability = $index_exists ? 'fulltext' : 'like';
2591	update_option('mxchat_hybrid_keyword_capability', $capability);
2592	return $capability;
2593	}
2594
2595	/**
2596	* Public entry for re-embedding already-stored content in place (wp mxchat
2597	* rtl-repair, plan d1e6f7). Thin wrapper so the repair CLI gets the exact
2598	* provider routing the import path uses — the repaired vector must come from
2599	* the same model family the bot indexes with, or retrieval stays broken.
2600	*/
2601	public static function regenerate_embedding($text, $api_key, $bot_id = 'default') {
2602	return self::generate_embedding($text, $api_key, $bot_id);
2603	}
2604
2605	/**
2606	* Restore logical character order in PDF-extracted RTL text (plan 32bf9e).
2607	*
2608	* The bundled Smalot parser only un-reverses text runs tagged with the
2609	* ReversedChars marked-content operator (Word emits it; LibreOffice and most
2610	* other producers do not), so their Hebrew/Arabic PDFs extract in visual
2611	* (reversed) order and embed/search as garbage. This is OUR post-processing
2612	* seam over getText() — the parser itself is never patched (it gets replaced
2613	* wholesale on library updates).
2614	*
2615	* Heuristic and deliberately conservative, per line:
2616	* - lines without strong RTL codepoints are untouched (a fully-Latin line in
2617	* an RTL document therefore stays as extracted — accepted limitation);
2618	* - Arabic presentation forms are a definitive visual-order signal (they only
2619	* appear in shaped output): de-shape to base letters and reverse;
2620	* - otherwise flip only on positive evidence — Hebrew final-letter position
2621	* (a sofit at word START only happens in reversed text) or sentence
2622	* punctuation position (leading in visual order, trailing in logical);
2623	* - ambiguous lines are left alone: a conservative miss beats corrupting a
2624	* Word-produced extraction the parser already handled (the double-flip
2625	* guard this plan's approval named mandatory).
2626	*
2627	* @param string $text One extracted page string, straight from getText().
2628	* @param string $context Caller tag for the Debug Mode entry (site + page).
2629	* @return string Text with RTL lines restored to logical order.
2630	*/
2631	public static function normalize_pdf_rtl($text, $context = '') {
2632	if (!is_string($text) \|\| '' === $text) {
2633	return $text;
2634	}
2635	// Escape hatch for sites whose PDFs already extract logically.
2636	if (!apply_filters('mxchat_pdf_rtl_normalize', true, $text)) {
2637	return $text;
2638	}
2639	// Fast bail: nothing RTL anywhere in the page.
2640	if (!preg_match('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $text)) {
2641	return $text;
2642	}
2643
2644	$parts = preg_split('/(\R)/u', $text, -1, PREG_SPLIT_DELIM_CAPTURE);
2645	if (false === $parts) {
2646	return $text;
2647	}
2648
2649	$flipped_lines = 0;
2650	$deshaped_lines = 0;
2651	foreach ($parts as $i => $part) {
2652	if ('' === $part \|\| preg_match('/^\R$/u', $part)) {
2653	continue;
2654	}
2655	$was_flipped = false;
2656	$was_deshaped = false;
2657	$new = self::pdf_rtl_normalize_line($part, $was_flipped, $was_deshaped);
2658	if ($new !== $part) {
2659	$parts[$i] = $new;
2660	}
2661	if ($was_flipped) {
2662	$flipped_lines++;
2663	}
2664	if ($was_deshaped) {
2665	$deshaped_lines++;
2666	}
2667	}
2668
2669	if (($flipped_lines \|\| $deshaped_lines) && class_exists('MxChat_Admin')) {
2670	MxChat_Admin::mxchat_log_debug('pdf_rtl_normalized', 'RTL PDF text restored to logical order', array(
2671	'context' => (string) $context,
2672	'lines_flipped' => $flipped_lines,
2673	'lines_deshaped' => $deshaped_lines,
2674	'decision' => 'visual-order extraction detected',
2675	));
2676	}
2677
2678	return implode('', $parts);
2679	}
2680
2681	/**
2682	* Normalize one line. Sets $flipped/$deshaped for the caller's debug entry.
2683	*/
2684	private static function pdf_rtl_normalize_line($line, &$flipped, &$deshaped) {
2685	$flipped = false;
2686	$deshaped = false;
2687
2688	if (!preg_match('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $line)) {
2689	return $line;
2690	}
2691
2692	$has_forms = (bool) preg_match('/[\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $line);
2693	$work = $line;
2694	if ($has_forms) {
2695	$work = strtr($work, self::pdf_rtl_deshape_map());
2696	$deshaped = ($work !== $line);
2697	}
2698
2699	$verdict = 'ambiguous';
2700	if ($has_forms) {
2701	// Shaped glyph codepoints only exist in visual-order output.
2702	$verdict = 'visual';
2703	} else {
2704	// Strong-direction dominance gate first: an LTR-dominant line with an
2705	// embedded RTL word is not flip material.
2706	$rtl_count = preg_match_all('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}]/u', $work, $m_rtl);
2707	$ltr_count = preg_match_all('/[A-Za-z]/u', $work, $m_ltr);
2708	if ($rtl_count < 1 \|\| $rtl_count <= $ltr_count) {
2709	return $line;
2710	}
2711
2712	// Hebrew final letters (ך ם ן ף ץ) end words in logical text; one at
2713	// a word START (Hebrew letter follows, none precedes) is reversal
2714	// evidence. Positional, so it survives the line being reversed.
2715	$sofit_initial = preg_match_all('/(?<![\x{05D0}-\x{05EA}])[\x{05DA}\x{05DD}\x{05DF}\x{05E3}\x{05E5}](?=[\x{05D0}-\x{05EA}])/u', $work, $m_i);
2716	$sofit_terminal = preg_match_all('/(?<=[\x{05D0}-\x{05EA}])[\x{05DA}\x{05DD}\x{05DF}\x{05E3}\x{05E5}](?![\x{05D0}-\x{05EA}])/u', $work, $m_t);
2717	if ($sofit_initial > $sofit_terminal) {
2718	$verdict = 'visual';
2719	} elseif ($sofit_terminal > $sofit_initial) {
2720	$verdict = 'logical';
2721	} else {
2722	// Sentence punctuation lands at the visual LEFT edge of an RTL
2723	// line, i.e. the START of a visual-order extraction.
2724	$trimmed = trim($work);
2725	$starts_punct = (bool) preg_match('/^[.?!:;,]/u', $trimmed);
2726	$ends_punct = (bool) preg_match('/[.?!:;,]$/u', $trimmed);
2727	if ($starts_punct && !$ends_punct) {
2728	$verdict = 'visual';
2729	} elseif ($ends_punct && !$starts_punct) {
2730	$verdict = 'logical';
2731	}
2732	}
2733	}
2734
2735	if ('visual' !== $verdict) {
2736	// Ambiguous or logical: hand back the original line UNLESS we
2737	// de-shaped (de-shaping alone is always safe — same letters, same
2738	// order, un-ligated).
2739	return $deshaped ? $work : $line;
2740	}
2741
2742	$flipped = true;
2743	return self::pdf_rtl_flip_line($work);
2744	}
2745
2746	/**
2747	* Reverse a visual-order line into logical order: full character reversal,
2748	* mirror paired punctuation, then re-reverse embedded LTR runs (Latin words
2749	* and digit sequences, incl. Arabic-Indic digits) so they stay readable.
2750	*/
2751	private static function pdf_rtl_flip_line($line) {
2752	$chars = preg_split('//u', $line, -1, PREG_SPLIT_NO_EMPTY);
2753	if (false === $chars) {
2754	return $line;
2755	}
2756	$reversed = implode('', array_reverse($chars));
2757	$reversed = strtr($reversed, array(
2758	'(' => ')', ')' => '(',
2759	'[' => ']', ']' => '[',
2760	'{' => '}', '}' => '{',
2761	'<' => '>', '>' => '<',
2762	));
2763	$restored = preg_replace_callback(
2764	'/[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9}](?:[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9} .,\'"%\-:\/]*[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9}])?/u',
2765	function ($m) {
2766	$run = preg_split('//u', $m[0], -1, PREG_SPLIT_NO_EMPTY);
2767	return false === $run ? $m[0] : implode('', array_reverse($run));
2768	},
2769	$reversed
2770	);
2771	return null === $restored ? $reversed : $restored;
2772	}
2773
2774	/**
2775	* Arabic presentation forms (A + B) -> base letters. Built once from range
2776	* specs rather than ~120 hand-written literal entries; every codepoint in a
2777	* range maps to the same base sequence (isolated/final/initial/medial forms
2778	* of one letter are contiguous in the FE70 block).
2779	*/
2780	private static function pdf_rtl_deshape_map() {
2781	static $map = null;
2782	if (null !== $map) {
2783	return $map;
2784	}
2785	$ranges = array(
2786	// Form B harakat (each pair = standalone + tatweel-joined form).
2787	array(0xFE70, 0xFE71, array(0x064B)), array(0xFE72, 0xFE72, array(0x064C)),
2788	array(0xFE74, 0xFE74, array(0x064D)), array(0xFE76, 0xFE77, array(0x064E)),
2789	array(0xFE78, 0xFE79, array(0x064F)), array(0xFE7A, 0xFE7B, array(0x0650)),
2790	array(0xFE7C, 0xFE7D, array(0x0651)), array(0xFE7E, 0xFE7F, array(0x0652)),
2791	// Form B letters.
2792	array(0xFE80, 0xFE80, array(0x0621)), array(0xFE81, 0xFE82, array(0x0622)),
2793	array(0xFE83, 0xFE84, array(0x0623)), array(0xFE85, 0xFE86, array(0x0624)),
2794	array(0xFE87, 0xFE88, array(0x0625)), array(0xFE89, 0xFE8C, array(0x0626)),
2795	array(0xFE8D, 0xFE8E, array(0x0627)), array(0xFE8F, 0xFE92, array(0x0628)),
2796	array(0xFE93, 0xFE94, array(0x0629)), array(0xFE95, 0xFE98, array(0x062A)),
2797	array(0xFE99, 0xFE9C, array(0x062B)), array(0xFE9D, 0xFEA0, array(0x062C)),
2798	array(0xFEA1, 0xFEA4, array(0x062D)), array(0xFEA5, 0xFEA8, array(0x062E)),
2799	array(0xFEA9, 0xFEAA, array(0x062F)), array(0xFEAB, 0xFEAC, array(0x0630)),
2800	array(0xFEAD, 0xFEAE, array(0x0631)), array(0xFEAF, 0xFEB0, array(0x0632)),
2801	array(0xFEB1, 0xFEB4, array(0x0633)), array(0xFEB5, 0xFEB8, array(0x0634)),
2802	array(0xFEB9, 0xFEBC, array(0x0635)), array(0xFEBD, 0xFEC0, array(0x0636)),
2803	array(0xFEC1, 0xFEC4, array(0x0637)), array(0xFEC5, 0xFEC8, array(0x0638)),
2804	array(0xFEC9, 0xFECC, array(0x0639)), array(0xFECD, 0xFED0, array(0x063A)),
2805	array(0xFED1, 0xFED4, array(0x0641)), array(0xFED5, 0xFED8, array(0x0642)),
2806	array(0xFED9, 0xFEDC, array(0x0643)), array(0xFEDD, 0xFEE0, array(0x0644)),
2807	array(0xFEE1, 0xFEE4, array(0x0645)), array(0xFEE5, 0xFEE8, array(0x0646)),
2808	array(0xFEE9, 0xFEEC, array(0x0647)), array(0xFEED, 0xFEEE, array(0x0648)),
2809	array(0xFEEF, 0xFEF0, array(0x0649)), array(0xFEF1, 0xFEF4, array(0x064A)),
2810	// Form B lam-alef ligatures decompose to two letters.
2811	array(0xFEF5, 0xFEF6, array(0x0644, 0x0622)), array(0xFEF7, 0xFEF8, array(0x0644, 0x0623)),
2812	array(0xFEF9, 0xFEFA, array(0x0644, 0x0625)), array(0xFEFB, 0xFEFC, array(0x0644, 0x0627)),
2813	// Form A: Persian / Urdu letters in common use.
2814	array(0xFB56, 0xFB59, array(0x067E)), array(0xFB66, 0xFB69, array(0x0679)),
2815	array(0xFB7A, 0xFB7D, array(0x0686)), array(0xFB88, 0xFB89, array(0x0688)),
2816	array(0xFB8A, 0xFB8B, array(0x0698)), array(0xFB8E, 0xFB91, array(0x06A9)),
2817	array(0xFB92, 0xFB95, array(0x06AF)), array(0xFBA6, 0xFBA9, array(0x06C1)),
2818	array(0xFBAA, 0xFBAD, array(0x06BE)), array(0xFBAE, 0xFBAF, array(0x06D2)),
2819	array(0xFBFC, 0xFBFF, array(0x06CC)),
2820	);
2821	$map = array();
2822	foreach ($ranges as $range) {
2823	$base = '';
2824	foreach ($range[2] as $cp) {
2825	$base .= self::pdf_rtl_cp_to_utf8($cp);
2826	}
2827	for ($cp = $range[0]; $cp <= $range[1]; $cp++) {
2828	$map[self::pdf_rtl_cp_to_utf8($cp)] = $base;
2829	}
2830	}
2831	return $map;
2832	}
2833
2834	/**
2835	* Codepoint to UTF-8 without ext-intl / mbstring entity tricks (PHP 7.2 floor).
2836	*/
2837	private static function pdf_rtl_cp_to_utf8($cp) {
2838	if ($cp < 0x80) {
2839	return chr($cp);
2840	}
2841	if ($cp < 0x800) {
2842	return chr(0xC0 \| ($cp >> 6)) . chr(0x80 \| ($cp & 0x3F));
2843	}
2844	if ($cp < 0x10000) {
2845	return chr(0xE0 \| ($cp >> 12)) . chr(0x80 \| (($cp >> 6) & 0x3F)) . chr(0x80 \| ($cp & 0x3F));
2846	}
2847	return chr(0xF0 \| ($cp >> 18)) . chr(0x80 \| (($cp >> 12) & 0x3F)) . chr(0x80 \| (($cp >> 6) & 0x3F)) . chr(0x80 \| ($cp & 0x3F));
2848	}
2849	}