← All changes | includes/class-mxchat-utils.php +1670 -73 3.2.16 → 3.2.25 View file →

		@@ -5,8 +5,17 @@
5	5
6	6	class MxChat_Utils {
7	7
8	8	/**
	9	+ * True while a storage routine is re-writing an entry it is about to re-add
	10	+ * (submit_chunked_content's clean-slate delete). delete_chunks_for_url() skips
	11	+ * the Vector Store mirror-delete while set — an internal re-store is not an
	12	+ * entry removal, and mirroring it would delete-then-reupload the entry's file
	13	+ * on every chunked save (plan 15b5c6).
	14	+ */
	15	+private static $vectorstore_mirror_suspended = false;
	16	+
	17	+/**
9	18	* Validate a client-supplied session id (plan-mxchat-20260731-d42bec).
10	19	*
11	20	* sanitize_text_field() — which every session_id read site used before this —
12	21	* preserves '/' and '..'. Harmless where the value is only an option or
		@@ -38,8 +47,336 @@
38	47	return preg_match('/\A[A-Za-z0-9_-]{1,128}\z/', $val) ? $val : '';
39	48	}
40	49
41	50	/**
	51	+ * Sanitize the lead-capture consent-checkbox label (plan b062c4).
	52	+ *
	53	+ * The label is owner-supplied and renders inside the widget's email form, so
	54	+ * this is a security boundary, not a formatting nicety. One explicit
	55	+ * allowlist, used at BOTH save time (options.php sanitize + autosave AJAX)
	56	+ * and render time (widget form, admin surfaces) so the two can never drift:
	57	+ * an anchor — the whole point is "I agree to the <a>Privacy Policy</a>" —
	58	+ * plus inline emphasis. No block tags, no images, no style attributes.
	59	+ *
	60	+ * The stored consent record keeps this exact sanitized string as "the text
	61	+ * the visitor saw", so it must be deterministic: same input, same output,
	62	+ * whichever path ran it.
	63	+ *
	64	+ * @param mixed $raw Owner-entered label.
	65	+ * @return string Sanitized label, capped at 1000 chars.
	66	+ */
	67	+public static function sanitize_consent_label($raw) {
	68	+ if (!is_scalar($raw)) {
	69	+ return '';
	70	+ }
	71	+
	72	+ $allowed = array(
	73	+ 'a' => array(
	74	+ 'href' => true,
	75	+ 'title' => true,
	76	+ 'target' => true,
	77	+ 'rel' => true,
	78	+ ),
	79	+ 'strong' => array(),
	80	+ 'em' => array(),
	81	+ 'br' => array(),
	82	+ );
	83	+
	84	+ $label = wp_kses(trim((string) $raw), $allowed);
	85	+
	86	+ return mb_substr($label, 0, 1000);
	87	+}
	88	+
	89	+/**
	90	+ * Per-request cache for get_session_history(). Mirrors get_option()'s
	91	+ * request-scoped caching, which the mxchat_history_ option reads got for
	92	+ * free before plan 839c4c moved history reads onto the transcripts table.
	93	+ */
	94	+private static $history_cache = array();
	95	+
	96	+/**
	97	+ * Session chat history read from the transcripts table, in the exact array
	98	+ * shape the legacy mxchat_history_<sid> option stored (plan 839c4c). The
	99	+ * option was a second copy of state the table already held — measured
	100	+ * byte-identical in role/content/order on 174 of 177 real sessions, with
	101	+ * the table a superset on the rest — at up to 64 KB per option row. The
	102	+ * table is now the single store; nothing writes the option any more.
	103	+ *
	104	+ * Shape notes, load-bearing for the consumers:
	105	+ * - id: the transcripts row id (int). Integer ids make the pollers'
	106	+ * ">" comparisons correct where the old uniqid() strings only worked by
	107	+ * accident of hex ordering.
	108	+ * - timestamp: milliseconds, derived from the table's second-resolution GMT
	109	+ * column (x1000). Consumers comparing against a real-millisecond client
	110	+ * cutoff MUST floor the cutoff to the second and err inclusive — see the
	111	+ * persistence-off filters in class-mxchat-integrator.php.
	112	+ * - agent_name: the row's user_identifier, which the writer sets to the
	113	+ * same displayed_name value the option carried (agent name when present,
	114	+ * else email, else identifier).
	115	+ *
	116	+ * Public and static so mxchat-woo / mxchat-forms can call the same accessor
	117	+ * as core, guarded with method_exists against an older mxchat-basic.
	118	+ *
	119	+ * @param string $session_id
	120	+ * @return array[] Chronological entries: id, role, content, timestamp, agent_name.
	121	+ */
	122	+public static function get_session_history($session_id) {
	123	+ global $wpdb;
	124	+
	125	+ $session_id = self::sanitize_session_id($session_id);
	126	+ if ($session_id === '') {
	127	+ return array();
	128	+ }
	129	+
	130	+ if (array_key_exists($session_id, self::$history_cache)) {
	131	+ return self::$history_cache[$session_id];
	132	+ }
	133	+
	134	+ $table = $wpdb->prefix . 'mxchat_chat_transcripts';
	135	+
	136	+ // No SHOW TABLES guard: this is the chat hot path and the table is
	137	+ // created on activation (with an admin-load safety net). A genuinely
	138	+ // missing table fails the query and yields the same empty history the
	139	+ // old option read produced on a fresh session.
	140	+ $rows = $wpdb->get_results(
	141	+ $wpdb->prepare(
	142	+ "SELECT id, role, message, user_identifier, timestamp
	143	+ FROM `$table` WHERE session_id = %s ORDER BY id ASC",
	144	+ $session_id
	145	+ ),
	146	+ ARRAY_A
	147	+ );
	148	+
	149	+ $history = array();
	150	+ if (is_array($rows)) {
	151	+ foreach ($rows as $row) {
	152	+ // The column stores GMT (current_time('mysql', 1) at the writer),
	153	+ // so pin the parse to UTC rather than the site timezone.
	154	+ $ts = strtotime($row['timestamp'] . ' +0000');
	155	+ $history[] = array(
	156	+ 'id' => (int) $row['id'],
	157	+ 'role' => (string) $row['role'],
	158	+ 'content' => (string) $row['message'],
	159	+ 'timestamp' => ($ts ? $ts : 0) * 1000,
	160	+ 'agent_name' => (string) $row['user_identifier'],
	161	+ );
	162	+ }
	163	+ }
	164	+
	165	+ self::$history_cache[$session_id] = $history;
	166	+
	167	+ return $history;
	168	+}
	169	+
	170	+/**
	171	+ * Drop the cached history for one session (or all). The writer calls this
	172	+ * after every insert so a later read in the same request — e.g. the AI
	173	+ * context build that follows saving the user's message — sees the new row,
	174	+ * matching the read-your-own-write behavior update_option() gave the old
	175	+ * option copy.
	176	+ *
	177	+ * @param string|null $session_id Null flushes everything (test seam).
	178	+ */
	179	+public static function flush_session_history_cache($session_id = null) {
	180	+ if ($session_id === null) {
	181	+ self::$history_cache = array();
	182	+ return;
	183	+ }
	184	+
	185	+ unset(self::$history_cache[(string) $session_id]);
	186	+}
	187	+
	188	+/**
	189	+ * Append one message to a session's transcript on behalf of an add-on
	190	+ * (plan 4a664e). The Forms add-on's wizard steps travel over the add-on's
	191	+ * own AJAX action, never through the chat request, so nothing wrote them
	192	+ * to the transcript: Transcripts, the live-agent handoff card, Download
	193	+ * Transcript and every transcript email showed "[WIZARD_COMPLETE]" and no
	194	+ * answers. This is the write half of get_session_history() — same row
	195	+ * shape as the chat writer (visitor identity, captured name/email and
	196	+ * originating page from the session store, GMT timestamp), without the
	197	+ * new-session side effects (ownership, notification email, delayed
	198	+ * transcript) that only the visitor's first message should trigger.
	199	+ *
	200	+ * Public and static so add-ons can call it, guarded with method_exists
	201	+ * against an older mxchat-basic.
	202	+ *
	203	+ * @param string $session_id Chat session id.
	204	+ * @param string $role user|bot|system|agent.
	205	+ * @param string $message Stored as-is: bot rows may carry HTML, user rows
	206	+ * are rendered as text by every reader.
	207	+ * @return int The new transcripts row id, 0 when nothing was written.
	208	+ */
	209	+public static function save_session_message($session_id, $role, $message) {
	210	+ global $wpdb;
	211	+
	212	+ $session_id = self::sanitize_session_id($session_id);
	213	+ $role = is_string($role) ? strtolower(trim($role)) : '';
	214	+ $message = is_scalar($message) ? trim((string) $message) : '';
	215	+
	216	+ if ($session_id === '' || $message === ''
	217	+ || !in_array($role, array('user', 'bot', 'system', 'agent'), true)) {
	218	+ return 0;
	219	+ }
	220	+
	221	+ $table = $wpdb->prefix . 'mxchat_chat_transcripts';
	222	+
	223	+ $saved_email = MxChat_Session_Store::get($session_id, 'email');
	224	+ $saved_name = MxChat_Session_Store::get($session_id, 'name');
	225	+
	226	+ $row = array(
	227	+ 'user_id' => is_user_logged_in() ? get_current_user_id() : 0,
	228	+ 'user_identifier' => MxChat_User::mxchat_get_user_identifier(),
	229	+ 'user_email' => $saved_email ?: MxChat_User::mxchat_get_user_email(),
	230	+ 'user_name' => $saved_name ?: '',
	231	+ 'session_id' => $session_id,
	232	+ 'role' => $role,
	233	+ 'message' => $message,
	234	+ 'timestamp' => current_time('mysql', 1),
	235	+ );
	236	+
	237	+ // Same page attribution as the rest of the session; the column pair was
	238	+ // added by an upgrade routine, so tolerate its absence like the chat
	239	+ // writer does.
	240	+ $originating = MxChat_Session_Store::get($session_id, 'originating_page');
	241	+ if (is_array($originating) && !empty($originating['url'])
	242	+ && $wpdb->get_var("SHOW COLUMNS FROM `$table` LIKE 'originating_page_url'")) {
	243	+ $row['originating_page_url'] = $originating['url'];
	244	+ $row['originating_page_title'] = isset($originating['title']) ? $originating['title'] : '';
	245	+ }
	246	+
	247	+ $inserted = $wpdb->insert($table, $row);
	248	+ if ($inserted === false) {
	249	+ return 0;
	250	+ }
	251	+
	252	+ $row_id = (int) $wpdb->insert_id;
	253	+ self::flush_session_history_cache($session_id);
	254	+
	255	+ return $row_id;
	256	+}
	257	+
	258	+/**
	259	+ * Most recipients the Notification Email field will accept (plan 2f131a).
	260	+ * A settings field is not a mailing list.
	261	+ */
	262	+const NOTIFICATION_EMAIL_MAX = 5;
	263	+
	264	+/**
	265	+ * Parse the Notification Email field into a list of recipients (plan 2f131a).
	266	+ *
	267	+ * THE TRAP THIS EXISTS TO CLOSE: sanitize_email() cannot be the validator for
	268	+ * this field, because its output for the failing input is VALID. WordPress
	269	+ * strips the separator and the surplus '@' and concatenates the remains:
	270	+ *
	271	+ * [email protected], [email protected] -> [email protected]
	272	+ *
	273	+ * and is_email() then returns true on that. So every guard in the plugin passed,
	274	+ * the address was stored, the autosave ticked green, and both the new-session
	275	+ * notification and the auto-emailed transcript went to a domain that does not
	276	+ * exist — with no error anywhere. Validating the RAW part BEFORE sanitizing is
	277	+ * the whole point; reversing those two lines silently restores the bug.
	278	+ *
	279	+ * All-or-nothing by design: if any entry is bad the caller must store NOTHING.
	280	+ * A partial accept — keeping the good addresses and dropping the bad one — is
	281	+ * the same defect in a new costume, because the owner still believes everyone
	282	+ * on their list is being notified.
	283	+ *
	284	+ * @param mixed $raw Raw field value, exactly as submitted.
	285	+ * @return array{emails: string[], error: string} Empty emails + empty error
	286	+ * means the field was empty.
	287	+ */
	288	+public static function parse_notification_emails($raw) {
	289	+ $out = array('emails' => array(), 'error' => '');
	290	+
	291	+ if (!is_scalar($raw)) {
	292	+ $out['error'] = __('The notification email could not be read.', 'mxchat');
	293	+ return $out;
	294	+ }
	295	+
	296	+ $raw = trim((string) $raw);
	297	+ if ($raw === '') {
	298	+ return $out; // genuinely empty — the caller falls back to admin_email
	299	+ }
	300	+
	301	+ $seen = array();
	302	+ foreach (preg_split('/[,;]/', $raw) as $part) {
	303	+ $part = trim($part);
	304	+ if ($part === '') {
	305	+ // A trailing or doubled separator carries no address, so skipping it
	306	+ // cannot silently drop a recipient. This is the ONLY thing tolerated.
	307	+ continue;
	308	+ }
	309	+
	310	+ // RAW first. See the note above — order is load-bearing.
	311	+ $clean = is_email($part) ? sanitize_email($part) : '';
	312	+ if ($clean === '' || !is_email($clean)) {
	313	+ return array(
	314	+ 'emails' => array(),
	315	+ 'error' => sprintf(
	316	+ /* translators: %s: the email address the owner typed. */
	317	+ __('"%s" is not a valid email address, so nothing was saved. Separate multiple addresses with a comma.', 'mxchat'),
	318	+ esc_html($part)
	319	+ ),
	320	+ );
	321	+ }
	322	+
	323	+ $key = strtolower($clean);
	324	+ if (isset($seen[$key])) {
	325	+ continue; // same address twice would simply mail them twice
	326	+ }
	327	+ $seen[$key] = true;
	328	+ $out['emails'][] = $clean;
	329	+ }
	330	+
	331	+ if (count($out['emails']) > self::NOTIFICATION_EMAIL_MAX) {
	332	+ return array(
	333	+ 'emails' => array(),
	334	+ 'error' => sprintf(
	335	+ /* translators: %d: maximum number of notification recipients. */
	336	+ __('Enter at most %d email addresses, separated by commas.', 'mxchat'),
	337	+ self::NOTIFICATION_EMAIL_MAX
	338	+ ),
	339	+ );
	340	+ }
	341	+
	342	+ return $out;
	343	+}
	344	+
	345	+/**
	346	+ * The stored recipient list, ready to hand to wp_mail() (plan 2f131a).
	347	+ *
	348	+ * Fallback rule, and it is narrow on purpose: an EMPTY field falls back to the
	349	+ * site admin address, because that is the documented behaviour and an owner who
	350	+ * never filled the field in still wants their notifications. A field holding
	351	+ * something unusable does NOT fall back — it sends nowhere, exactly as before
	352	+ * this plan. Falling back on bad input would mean a typo silently redirects a
	353	+ * store's transcripts to a different mailbox than the one on screen.
	354	+ *
	355	+ * @param array|null $options mxchat_transcripts_options, or null to read it.
	356	+ * @return string[] Recipients; empty means do not send.
	357	+ */
	358	+public static function notification_recipients($options = null) {
	359	+ if (!is_array($options)) {
	360	+ $options = get_option('mxchat_transcripts_options', array());
	361	+ if (!is_array($options)) {
	362	+ $options = array();
	363	+ }
	364	+ }
	365	+
	366	+ $raw = isset($options['mxchat_notification_email']) ? $options['mxchat_notification_email'] : '';
	367	+ $raw = is_scalar($raw) ? trim((string) $raw) : '';
	368	+
	369	+ if ($raw === '') {
	370	+ $admin = get_option('admin_email');
	371	+ return is_email($admin) ? array($admin) : array();
	372	+ }
	373	+
	374	+ $parsed = self::parse_notification_emails($raw);
	375	+ return $parsed['error'] === '' ? $parsed['emails'] : array();
	376	+}
	377	+
	378	+/**
42	379	* Centralized embedding model registry. Single source of truth for dimensions
43	380	* and provider, so model-switch protection logic doesn't drift across files.
44	381	*/
45	382	public static function embedding_model_registry() {
		@@ -57,8 +394,12 @@
57	394	return isset($registry[$model]) ? (int) $registry[$model]['dims'] : 0;
58	395	}
59	396
60	397	public static function embedding_model_label($model) {
	398	+ if (is_string($model) && strpos($model, 'custom:') === 0) {
	399	+ /* translators: %s: the embedding model name configured on the custom provider */
	400	+ return sprintf(__('%s (custom provider)', 'mxchat'), substr($model, 7));
	401	+ }
61	402	$registry = self::embedding_model_registry();
62	403	return isset($registry[$model]) ? $registry[$model]['label'] : $model;
63	404	}
64	405
		@@ -81,8 +422,135 @@
81	422	}
82	423	}
83	424
84	425	/**
	426	+ * The model name the custom-provider embedding path will send, mirroring the
	427	+ * fallback chain the request itself uses: dedicated custom embedding model,
	428	+ * else the custom chat model, else 'default'. Single source shared by
	429	+ * generate_embedding_custom() and the mismatch-warning "selected" side so the
	430	+ * two can never drift (plan ae02cb).
	431	+ */
	432	+public static function resolve_custom_embedding_model($options) {
	433	+ if (isset($options['custom_provider_embedding_model']) && trim((string) $options['custom_provider_embedding_model']) !== '') {
	434	+ return trim((string) $options['custom_provider_embedding_model']);
	435	+ }
	436	+ if (isset($options['custom_provider_model']) && trim((string) $options['custom_provider_model']) !== '') {
	437	+ return trim((string) $options['custom_provider_model']);
	438	+ }
	439	+ return 'default';
	440	+}
	441	+
	442	+/**
	443	+ * The EFFECTIVE selected embedding model — what the next embed will actually
	444	+ * use. With custom-provider embeddings on this is the custom identity in the
	445	+ * same 'custom:<model>' form stamp_active_embedding_model() records, not the
	446	+ * inert standard dropdown value. Mismatch-warning comparisons must read this,
	447	+ * never $options['embedding_model'] directly — the dropdown cannot be
	448	+ * deselected, so reading it raw flags every correctly-configured custom setup.
	449	+ */
	450	+public static function get_selected_embedding_model($options = null) {
	451	+ if (!is_array($options)) {
	452	+ $options = get_option('mxchat_options', array());
	453	+ }
	454	+ if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
	455	+ return 'custom:' . self::resolve_custom_embedding_model($options);
	456	+ }
	457	+ return $options['embedding_model'] ?? '';
	458	+}
	459	+
	460	+/**
	461	+ * Dimension of a STANDARD embedding model, from the dropdown and its two
	462	+ * adjustable-size settings. Says nothing about a Custom Provider model — ask
	463	+ * expected_embedding_dimension() for the model the site actually embeds with.
	464	+ */
	465	+public static function standard_embedding_dimension($options = null) {
	466	+ if (!is_array($options)) {
	467	+ $options = get_option('mxchat_options', array());
	468	+ $options = is_array($options) ? $options : array();
	469	+ }
	470	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	471	+ $model_dimensions = array(
	472	+ 'text-embedding-ada-002' => 1536,
	473	+ 'text-embedding-3-small' => 1536,
	474	+ 'text-embedding-3-large' => 3072,
	475	+ 'voyage-2' => 1024,
	476	+ 'voyage-large-2' => 1536,
	477	+ 'voyage-3-large' => 2048,
	478	+ 'gemini-embedding-001' => 1536,
	479	+ );
	480	+ if (strpos($selected_model, 'voyage-3-large') === 0) {
	481	+ return intval($options['voyage_output_dimension'] ?? 2048);
	482	+ }
	483	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	484	+ return intval($options['gemini_output_dimension'] ?? 1536);
	485	+ }
	486	+ return $model_dimensions[$selected_model] ?? 1536;
	487	+}
	488	+
	489	+/**
	490	+ * Dimension of the vectors the NEXT embed will produce — the one place that
	491	+ * answers it (plan 418afd). With Custom Provider embeddings on, the standard
	492	+ * dropdown is disabled and says nothing about the custom model, so the number
	493	+ * is read from the model itself: one short probe embed, remembered per base
	494	+ * URL + model. Three copies of the dropdown table used to answer instead, and
	495	+ * a custom model of another size got a wrong-sized index and wrong-sized
	496	+ * listing vectors.
	497	+ *
	498	+ * @param array|null $options mxchat_options (or a bot's merged options).
	499	+ * @param bool $force Probe again even when a recent probe failed.
	500	+ * @return int|WP_Error WP_Error only when a custom model's size could not be read.
	501	+ */
	502	+public static function expected_embedding_dimension($options = null, $force = false) {
	503	+ if (!is_array($options)) {
	504	+ $options = get_option('mxchat_options', array());
	505	+ $options = is_array($options) ? $options : array();
	506	+ }
	507	+ if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
	508	+ return self::custom_embedding_dimension($options, $force);
	509	+ }
	510	+ return self::standard_embedding_dimension($options);
	511	+}
	512	+
	513	+/**
	514	+ * A Custom Provider embedding model's dimension, by asking it once. The
	515	+ * answer is stored against the base URL + model, so changing either asks
	516	+ * again. A failure is remembered for five minutes so a dead endpoint does not
	517	+ * hold up every admin page that prints the number; $force skips that memory.
	518	+ *
	519	+ * @return int|WP_Error
	520	+ */
	521	+public static function custom_embedding_dimension($options, $force = false) {
	522	+ $model = self::resolve_custom_embedding_model($options);
	523	+ $base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
	524	+ $key = md5(strtolower($base_url) . '|' . $model);
	525	+
	526	+ $known = get_option('mxchat_custom_embedding_dimension', array());
	527	+ if (is_array($known) && ($known['key'] ?? '') === $key && (int) ($known['dimension'] ?? 0) > 0) {
	528	+ return (int) $known['dimension'];
	529	+ }
	530	+
	531	+ $failed = get_transient('mxchat_custom_embedding_dimension_failed');
	532	+ if (!$force && is_array($failed) && ($failed['key'] ?? '') === $key) {
	533	+ return new WP_Error('embedding_dimension_unknown', (string) ($failed['reason'] ?? ''));
	534	+ }
	535	+
	536	+ $vector = self::generate_embedding_custom('MxChat embedding dimension check', $options);
	537	+ if (!is_array($vector) || count($vector) === 0) {
	538	+ $reason = (is_string($vector) && $vector !== '') ? $vector : __('The custom provider returned no embedding.', 'mxchat');
	539	+ set_transient('mxchat_custom_embedding_dimension_failed', array('key' => $key, 'reason' => $reason), 5 * MINUTE_IN_SECONDS);
	540	+ return new WP_Error('embedding_dimension_unknown', $reason);
	541	+ }
	542	+
	543	+ update_option('mxchat_custom_embedding_dimension', array(
	544	+ 'key' => $key,
	545	+ 'model' => 'custom:' . $model,
	546	+ 'dimension' => count($vector),
	547	+ ), false);
	548	+ delete_transient('mxchat_custom_embedding_dimension_failed');
	549	+ return count($vector);
	550	+}
	551	+
	552	+/**
85	553	* Extract the 11-character YouTube video ID from a URL, or '' if the URL is
86	554	* not a single-video YouTube link. Single source of truth for both the KB
87	555	* ingestion side and the chat render side — do not duplicate this parsing.
88	556	* Channel, playlist, and search URLs deliberately return '' (only a URL that
		@@ -111,8 +579,45 @@
111	579	return (strlen($id) === 11) ? $id : '';
112	580	}
113	581
114	582	/**
	583	+ * plan-mxchat-20260813-f52492 — video-card gating.
	584	+ *
	585	+ * Two standalone options (NOT mxchat_options — they skip the sanitize/autosave
	586	+ * traps entirely), read here so the gate in the integrator and the fields on
	587	+ * Knowledge -> Chunking & Retrieval can never disagree about a default.
	588	+ *
	589	+ * Master switch. Default ON: the card is existing behavior, and this is an
	590	+ * opt-out for owners who never want one, not a new feature to opt into.
	591	+ */
	592	+public static function video_embed_enabled() {
	593	+ return get_option('mxchat_video_embed_enabled', 'on') === 'on';
	594	+}
	595	+
	596	+/**
	597	+ * The video card's OWN confidence floor, as a 0-1 cosine — deliberately not
	598	+ * the site-wide Similarity Threshold (default 35). "Good enough to quote in
	599	+ * the answer" and "good enough to put a video on screen" are different
	600	+ * questions: retrieval is allowed to be generous because the model still
	601	+ * decides what to say, whereas the card is asserted to the visitor with no
	602	+ * such filter. Stored as an int percentage to match the site-wide slider's
	603	+ * convention; the default (55) sits above it on purpose.
	604	+ *
	605	+ * MXCHAT_VIDEO_EMBED_THRESHOLD_DEFAULT is the single source of that number.
	606	+ */
	607	+public static function video_embed_threshold() {
	608	+ $stored = get_option('mxchat_video_embed_threshold', null);
	609	+ $percent = ($stored === null || $stored === '')
	610	+ ? MXCHAT_VIDEO_EMBED_THRESHOLD_DEFAULT
	611	+ : (int) $stored;
	612	+ if ($percent < 0) { $percent = 0; }
	613	+ if ($percent > 100) { $percent = 100; }
	614	+ // Cast: PHP evaluates 100/100 to int(1), so an unclamped return type would
	615	+ // vary with the stored value. Callers compare against a cosine — keep it float.
	616	+ return (float) $percent / 100;
	617	+}
	618	+
	619	+/**
115	620	* UPDATED: Submit or update content (and its embedding) in the database.
116	621	* Stores in Pinecone if enabled, otherwise stores in WordPress DB.
117	622	*
118	623	* @param string $content The content to be embedded.
		@@ -122,9 +627,9 @@
122	627	* @param string $bot_id The bot ID for multi-bot support
123	628	* @param string $content_type The type of content (post, page, pdf, url, manual, product, etc.)
124	629	* @return bool|WP_Error True on success, WP_Error on failure
125	630	*/
126		-public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
	631	+public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content', $title = '') {
127	632	global $wpdb;
128	633	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
129	634
130	635	//error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ', Type: ' . $content_type . ')');
		@@ -129,10 +634,18 @@
129	634
130	635	//error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ', Type: ' . $content_type . ')');
131	636	//error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
132	637
133		- // Sanitize the source URL
134		- $source_url = esc_url_raw($source_url);
	638	+ // Sanitize the source URL. Internal identities (mxchat:// manual docs,
	639	+ // upload:// file uploads) are NOT web URLs — esc_url_raw EMPTIES them
	640	+ // because its protocol list is statically cached and effectively
	641	+ // unfilterable (measured, plan 945406 / 0485e5) — sanitize those as text
	642	+ // so upserts stay keyed to a stable identity across re-imports.
	643	+ if (preg_match('#^(mxchat|upload)://#i', $source_url)) {
	644	+ $source_url = sanitize_text_field($source_url);
	645	+ } else {
	646	+ $source_url = esc_url_raw($source_url);
	647	+ }
135	648
136	649	// Sanitize content_type
137	650	$content_type = sanitize_key($content_type);
138	651	if (empty($content_type)) {
		@@ -138,8 +651,14 @@
138	651	if (empty($content_type)) {
139	652	$content_type = 'content'; // Fallback for backwards compatibility
140	653	}
141	654
	655	+ // Page title captured by the importer (plan d9ee66), registered under the
	656	+ // SANITIZED url so the storage writers below find it by the same key.
	657	+ if (is_string($title) && trim($title) !== '') {
	658	+ self::set_source_title($source_url, $title);
	659	+ }
	660	+
142	661	// Just ensure UTF-8 validity without aggressive escaping
143	662	$safe_content = wp_check_invalid_utf8($content);
144	663	// Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D)
145	664	$safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
		@@ -147,9 +666,16 @@
147	666	// Check if chunking should be applied
148	667	$chunker = MxChat_Chunker::from_settings();
149	668	if ($chunker->should_chunk($safe_content)) {
150	669	//error_log('[MXCHAT-DB] Content exceeds chunk threshold, using chunked submission');
151		- return self::submit_chunked_content($safe_content, $source_url, $api_key, $bot_id, $content_type, $chunker);
	670	+ $chunk_result = self::submit_chunked_content($safe_content, $source_url, $api_key, $bot_id, $content_type, $chunker);
	671	+ // Vector Store mirror gets the entry WHOLE — one file per KB entry,
	672	+ // OpenAI chunks server-side; local chunking is our own embedding
	673	+ // concern and never reaches the store (plan 15b5c6).
	674	+ if ($chunk_result === true && class_exists('MxChat_Vectorstore_Manager')) {
	675	+ MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type);
	676	+ }
	677	+ return $chunk_result;
152	678	}
153	679
154	680	// UPDATED: Generate the embedding using bot-specific configuration
155	681	$embedding_vector = self::generate_embedding($content, $api_key, $bot_id);
		@@ -167,18 +693,175 @@
167	693	// UPDATED: Check if Pinecone is enabled for this specific bot
168	694	if (self::is_pinecone_enabled_for_bot($bot_id)) {
169	695	//error_log('[MXCHAT-DB] Pinecone is enabled for bot ' . $bot_id . ' - using Pinecone storage');
170	696	// Store in Pinecone only
171		- return self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id, $content_type);
	697	+ $result = self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id, $content_type);
	698	+ // If this URL previously stored as CHUNKED content and the new content fits in a
	699	+ // single vector, the upsert above only overwrote the base id — the old
	700	+ // md5(url)_chunk_N vectors would keep serving the stale text. Sweep them.
	701	+ // Only for a real source_url: chunk ids derive from it, and md5('') is shared
	702	+ // by legacy URL-less entries so a blind sweep there could hit other entries.
	703	+ if ($result === true && !empty($source_url)) {
	704	+ self::cleanup_pinecone_chunk_stragglers($source_url, $bot_id);
	705	+ }
	706	+ if ($result === true && class_exists('MxChat_Vectorstore_Manager')) {
	707	+ MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type);
	708	+ }
	709	+ return $result;
172	710	} else {
173	711	//error_log('[MXCHAT-DB] Pinecone not enabled for bot ' . $bot_id . ' - using WordPress storage');
174	712	// Store in WordPress database only
175	713	$embedding_vector_serialized = maybe_serialize($embedding_vector);
176		- return self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type);
	714	+ $result = self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type);
	715	+ if ($result === true && class_exists('MxChat_Vectorstore_Manager')) {
	716	+ MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type);
	717	+ }
	718	+ return $result;
177	719	}
178	720	}
179	721
	722	+/* -------------------------------------------------------------------------
	723	+ * Source titles (3.2.23, plan d9ee66)
	724	+ *
	725	+ * A knowledge entry imported from a URL or sitemap remembers the page's own
	726	+ * title, so an add-on that names a source (AI Search's source cards) can show
	727	+ * "About Us" instead of a guess from the slug ("Aboutus"). Stored in the
	728	+ * `title` column of the WP knowledge table and as `title` metadata on Pinecone
	729	+ * vectors / documents. The importer hands the title to submit_content_to_db(),
	730	+ * which registers it under the source URL; the writers look it up by that key.
	731	+ * ---------------------------------------------------------------------- */
	732	+
	733	+/** @var array<string,string> source_url => title, for the current request */
	734	+private static $source_titles = array();
	735	+
	736	+public static function set_source_title($source_url, $title) {
	737	+ $key = trim((string) $source_url);
	738	+ $title = self::clean_title($title);
	739	+ if ($key === '' || $title === '') {
	740	+ return;
	741	+ }
	742	+ self::$source_titles[$key] = $title;
	743	+}
	744	+
	745	+public static function source_title_for($source_url) {
	746	+ $key = trim((string) $source_url);
	747	+ return ($key !== '' && isset(self::$source_titles[$key])) ? self::$source_titles[$key] : '';
	748	+}
	749	+
	750	+/** Entities decoded, tags gone, whitespace collapsed, capped at the column width. */
	751	+public static function clean_title($title) {
	752	+ $title = html_entity_decode((string) $title, ENT_QUOTES | ENT_HTML5, 'UTF-8');
	753	+ $title = wp_strip_all_tags($title);
	754	+ $title = trim((string) preg_replace('/\s+/u', ' ', $title));
	755	+ if (mb_strlen($title, 'UTF-8') > 255) {
	756	+ $title = rtrim(mb_substr($title, 0, 255, 'UTF-8'));
	757	+ }
	758	+ return $title;
	759	+}
	760	+
180	761	/**
	762	+ * The page's own title from fetched HTML: og:title, else <title> with a
	763	+ * trailing "| Site Name" style suffix removed, else the first <h1>. Returns ''
	764	+ * when nothing usable is there, so the caller keeps its own fallback.
	765	+ */
	766	+public static function extract_page_title($html) {
	767	+ $html = (string) $html;
	768	+ if ($html === '') {
	769	+ return '';
	770	+ }
	771	+ $head = substr($html, 0, 200000);
	772	+ $site = '';
	773	+ if (preg_match('/<meta\s+[^>]*property=["\']og:site_name["\'][^>]*content=["\']([^"\']{1,120})["\']/i', $head, $m)
	774	+ || preg_match('/<meta\s+[^>]*content=["\']([^"\']{1,120})["\'][^>]*property=["\']og:site_name["\']/i', $head, $m)) {
	775	+ $site = self::clean_title($m[1]);
	776	+ }
	777	+ $candidate = '';
	778	+ if (preg_match('/<meta\s+[^>]*property=["\']og:title["\'][^>]*content=["\']([^"\']{1,500})["\']/i', $head, $m)
	779	+ || preg_match('/<meta\s+[^>]*content=["\']([^"\']{1,500})["\'][^>]*property=["\']og:title["\']/i', $head, $m)) {
	780	+ $candidate = self::clean_title($m[1]);
	781	+ }
	782	+ if ($candidate === '' && preg_match('/<title[^>]*>(.*?)<\/title>/is', $head, $m)) {
	783	+ $candidate = self::clean_title($m[1]);
	784	+ }
	785	+ if ($candidate !== '') {
	786	+ $candidate = self::strip_site_suffix($candidate, $site);
	787	+ }
	788	+ if ($candidate === '' && preg_match('/<h1[^>]*>(.*?)<\/h1>/is', $html, $m)) {
	789	+ $candidate = self::clean_title($m[1]);
	790	+ }
	791	+ return $candidate;
	792	+}
	793	+
	794	+/**
	795	+ * "About Us | Just Facts" -> "About Us". Only a TRAILING segment after a spaced
	796	+ * separator goes, only when what is left is a real title (3+ characters) and the
	797	+ * removed part is short enough to be a site name; a known og:site_name wins.
	798	+ * Byte offsets throughout: every separator starts with an ASCII space, so a cut
	799	+ * there never lands inside a multibyte character.
	800	+ */
	801	+private static function strip_site_suffix($title, $site_name = '') {
	802	+ $seps = array(' | ', ' – ', ' — ', ' - ', ' :: ', ' · ', ' » ', ' • ');
	803	+ if ($site_name !== '') {
	804	+ foreach ($seps as $sep) {
	805	+ $tail = $sep . $site_name;
	806	+ if (strlen($title) > strlen($tail) && substr($title, -strlen($tail)) === $tail) {
	807	+ return trim(substr($title, 0, strlen($title) - strlen($tail)));
	808	+ }
	809	+ }
	810	+ }
	811	+ $best_pos = -1;
	812	+ $best_sep = '';
	813	+ foreach ($seps as $sep) {
	814	+ $pos = strrpos($title, $sep);
	815	+ if ($pos !== false && $pos > $best_pos) {
	816	+ $best_pos = $pos;
	817	+ $best_sep = $sep;
	818	+ }
	819	+ }
	820	+ if ($best_pos > 0) {
	821	+ $left = trim(substr($title, 0, $best_pos));
	822	+ $right = trim(substr($title, $best_pos + strlen($best_sep)));
	823	+ if (mb_strlen($left, 'UTF-8') >= 3 && $right !== '' && mb_strlen($right, 'UTF-8') <= 60) {
	824	+ return $left;
	825	+ }
	826	+ }
	827	+ return $title;
	828	+}
	829	+
	830	+/**
	831	+ * Make sure the knowledge table has the `title` column (3.2.23, plan d9ee66).
	832	+ * Activation adds it; this is the safety net for a copy updated in place, and
	833	+ * the one place add-ons ask "may I read/write title?". A positive answer is
	834	+ * remembered in an option, so after the first time this costs one option read.
	835	+ */
	836	+public static function ensure_kb_title_column() {
	837	+ global $wpdb;
	838	+ static $ready = null;
	839	+ if ($ready !== null) {
	840	+ return $ready;
	841	+ }
	842	+ if (get_option('mxchat_kb_title_column') === '1') {
	843	+ $ready = true;
	844	+ return true;
	845	+ }
	846	+ $table = $wpdb->prefix . 'mxchat_system_prompt_content';
	847	+ $cols = $wpdb->get_col("SHOW COLUMNS FROM {$table}", 0);
	848	+ if (empty($cols)) {
	849	+ $ready = false;
	850	+ return false;
	851	+ }
	852	+ if (!in_array('title', $cols, true)) {
	853	+ $wpdb->query("ALTER TABLE {$table} ADD COLUMN title VARCHAR(255) DEFAULT NULL AFTER source_url");
	854	+ $cols = $wpdb->get_col("SHOW COLUMNS FROM {$table}", 0);
	855	+ }
	856	+ $ready = in_array('title', $cols, true);
	857	+ if ($ready) {
	858	+ update_option('mxchat_kb_title_column', '1', false);
	859	+ }
	860	+ return $ready;
	861	+}
	862	+
	863	+/**
181	864	* UPDATED: Check if Pinecone is enabled and properly configured for a specific bot
182	865	*/
183	866	private static function is_pinecone_enabled_for_bot($bot_id = 'default') {
184	867	// For default bot or when multi-bot is not active, use original method
		@@ -290,11 +973,15 @@
290	973	// Check if this is truly manual content (no URL at all) vs a real URL that filter_var rejects
291	974	// filter_var(FILTER_VALIDATE_URL) rejects valid URLs with encoded chars, non-ASCII, fragments, etc.
292	975	// Use a looser check: if it starts with http(s):// or has a scheme, it's a URL
293	976	$has_url_scheme = !empty($source_url) && preg_match('#^https?://#i', $source_url);
	977	+ // upload:// identities (admin document/PDF uploads, plan 0485e5) are stable
	978	+ // and deduplicable — treat them like URLs so a re-upload UPDATES the row
	979	+ // instead of minting a fresh manual identity (which would duplicate).
	980	+ $has_stable_identity = $has_url_scheme || (!empty($source_url) && preg_match('#^upload://#i', $source_url));
294	981	// Treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
295	982	$is_legacy_mxchat_url = $has_url_scheme && strpos($source_url, 'mxchat.ai') !== false;
296		- $is_manual_content = empty($source_url) || $source_url === '' || !$has_url_scheme || $is_legacy_mxchat_url;
	983	+ $is_manual_content = empty($source_url) || $source_url === '' || !$has_stable_identity || $is_legacy_mxchat_url;
297	984
298	985	if ($is_manual_content) {
299	986	// Generate unique identifier for manual content to prevent overwrites
300	987	$source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
		@@ -315,8 +1002,13 @@
315	1002	//error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)');
316	1003	}
317	1004	// ===== END FIX =====
318	1005
	1006	+ // Page title captured at import (plan d9ee66): written only when the
	1007	+ // importer supplied one, so a re-index without a title keeps the old value.
	1008	+ $title_value = $is_manual_content ? '' : self::source_title_for($source_url);
	1009	+ $title_ready = ($title_value !== '') && self::ensure_kb_title_column();
	1010	+
319	1011	// Progressive fallback mechanism for problematic content
320	1012	$attempt = 1;
321	1013	$max_attempts = 3;
322	1014	$current_content = $safe_content;
		@@ -327,39 +1019,41 @@
327	1019	if ($existing_id) {
328	1020	//error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
329	1021
330	1022	// Update the existing row - UPDATED 2.5.6: Added content_type
331		- $result = $wpdb->update(
332		- $table_name,
333		- array(
334		- 'url' => $source_url,
335		- 'article_content' => $current_content,
336		- 'embedding_vector' => $embedding_vector_serialized,
337		- 'source_url' => $source_url,
338		- 'content_type' => $content_type,
339		- 'timestamp' => current_time('mysql'),
340		- ),
341		- array('id' => $existing_id),
342		- array('%s','%s','%s','%s','%s','%s'),
343		- array('%d')
	1023	+ $row = array(
	1024	+ 'url' => $source_url,
	1025	+ 'article_content' => $current_content,
	1026	+ 'embedding_vector' => $embedding_vector_serialized,
	1027	+ 'source_url' => $source_url,
	1028	+ 'content_type' => $content_type,
	1029	+ 'timestamp' => current_time('mysql'),
344	1030	);
	1031	+ $formats = array('%s','%s','%s','%s','%s','%s');
	1032	+ if ($title_ready) {
	1033	+ $row['title'] = $title_value;
	1034	+ $formats[] = '%s';
	1035	+ }
	1036	+ $result = $wpdb->update($table_name, $row, array('id' => $existing_id), $formats, array('%d'));
345	1037	} else {
346	1038	//error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
347	1039	//error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
348	1040
349	1041	// Insert a new row - UPDATED 2.5.6: Added content_type
350		- $result = $wpdb->insert(
351		- $table_name,
352		- array(
353		- 'url' => $source_url, // Now unique for manual content
354		- 'article_content' => $current_content,
355		- 'embedding_vector' => $embedding_vector_serialized,
356		- 'source_url' => $source_url, // Now unique for manual content
357		- 'content_type' => $content_type,
358		- 'timestamp' => current_time('mysql'),
359		- ),
360		- array('%s','%s','%s','%s','%s','%s')
	1042	+ $row = array(
	1043	+ 'url' => $source_url, // Now unique for manual content
	1044	+ 'article_content' => $current_content,
	1045	+ 'embedding_vector' => $embedding_vector_serialized,
	1046	+ 'source_url' => $source_url, // Now unique for manual content
	1047	+ 'content_type' => $content_type,
	1048	+ 'timestamp' => current_time('mysql'),
361	1049	);
	1050	+ $formats = array('%s','%s','%s','%s','%s','%s');
	1051	+ if ($title_ready) {
	1052	+ $row['title'] = $title_value;
	1053	+ $formats[] = '%s';
	1054	+ }
	1055	+ $result = $wpdb->insert($table_name, $row, $formats);
362	1056	}
363	1057
364	1058	if ($result === false) {
365	1059	//error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . ')');
		@@ -418,10 +1112,11 @@
418	1112	// ===== UPDATED: Handle manual content with unique vector IDs =====
419	1113	if ($vector_id) {
420	1114	// Use provided vector ID
421	1115	//error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
422		- } elseif (!empty($url) && preg_match('#^https?://#i', $url)) {
423		- // For URLs, use URL-based ID (existing behavior)
	1116	+ } elseif (!empty($url) && preg_match('#^(https?|upload)://#i', $url)) {
	1117	+ // For URLs — and stable upload:// identities (plan 0485e5) — use an
	1118	+ // identity-derived ID so a re-import upserts the same vector.
424	1119	$vector_id = md5($url);
425	1120	//error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
426	1121	} else {
427	1122	// For manual content (empty/no URL scheme), generate unique ID
		@@ -482,9 +1177,23 @@
482	1177	'last_updated' => time(),
483	1178	'created_at' => time(), // Add creation timestamp
484	1179	'bot_id' => $bot_id, // Add bot identification
485	1180	);
486		-
	1181	+ $page_title = self::source_title_for($url);
	1182	+ if ($page_title !== '') {
	1183	+ $metadata['title'] = $page_title; // page title captured at import (plan d9ee66)
	1184	+ }
	1185	+
	1186	+ // Document index (plan 362c31): same id, same metadata, through the
	1187	+ // documents API. A site on the classic vector index never enters here.
	1188	+ if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) {
	1189	+ return MxChat_Pinecone_Documents::upsert_one($vector_id, $content, $embedding_vector, $metadata, array(
	1190	+ 'api_key' => $api_key,
	1191	+ 'host' => $host,
	1192	+ 'namespace' => $namespace,
	1193	+ ));
	1194	+ }
	1195	+
487	1196	$vector_data = array(
488	1197	'id' => $vector_id,
489	1198	'values' => $embedding_vector,
490	1199	'metadata' => $metadata
		@@ -539,8 +1248,104 @@
539	1248	return true;
540	1249	}
541	1250
542	1251	/**
	1252	+ * Caller-side pre-flight for KB ingestion: can an embedding request be made
	1253	+ * with these options, and which API key should travel downstream?
	1254	+ *
	1255	+ * Custom-provider-aware — generate_embedding() below routes to the custom
	1256	+ * endpoint FIRST and ignores the passed cloud key entirely when
	1257	+ * custom_provider_for_embeddings is on, so on that branch the only real
	1258	+ * requirement is a Base URL. Ingestion callers that gated on a cloud API key
	1259	+ * were killing keyless custom-embeddings sites (local Ollama / LM Studio
	1260	+ * class) before the embed layer could route (plan cbd5fd).
	1261	+ *
	1262	+ * NOTE: reads $options['embedding_model'] raw on purpose — this mirrors
	1263	+ * generate_embedding()'s own routing read, NOT the mismatch-banner's
	1264	+ * "selected" chain (get_selected_embedding_model). The helper must predict
	1265	+ * what the very next embed call will do, byte-for-byte.
	1266	+ *
	1267	+ * Decision only — callers keep their own error-surfacing shape (admin-notice
	1268	+ * transient + redirect, wp_send_json_error, WP_Error, silent return).
	1269	+ *
	1270	+ * @param array|null $options Resolved options (bot-specific where the caller
	1271	+ * has them); null loads the default bot's options.
	1272	+ * @return array {
	1273	+ * @type bool $ok Whether ingestion can proceed.
	1274	+ * @type string $api_key Key to pass downstream ('' on the custom branch —
	1275	+ * generate_embedding() ignores it there).
	1276	+ * @type string $reason Human-readable blocker; '' when $ok.
	1277	+ * @type string $provider Short provider label ('OpenAI', 'Voyage AI',
	1278	+ * 'Google Gemini', 'Custom Provider').
	1279	+ * }
	1280	+ */
	1281	+public static function embedding_preflight($options = null) {
	1282	+ if (!is_array($options)) {
	1283	+ $options = get_option('mxchat_options');
	1284	+ $options = is_array($options) ? $options : array();
	1285	+ }
	1286	+
	1287	+ // Custom branch mirrors generate_embedding()'s routing order (custom first).
	1288	+ if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
	1289	+ $base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
	1290	+ if ($base_url === '') {
	1291	+ return array(
	1292	+ 'ok' => false,
	1293	+ 'api_key' => '',
	1294	+ // Same string generate_embedding_custom() returns for this state.
	1295	+ 'reason' => __('Custom provider Base URL is not configured.', 'mxchat'),
	1296	+ 'provider' => 'Custom Provider',
	1297	+ );
	1298	+ }
	1299	+ return array('ok' => true, 'api_key' => '', 'reason' => '', 'provider' => 'Custom Provider');
	1300	+ }
	1301	+
	1302	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	1303	+ if (strpos($selected_model, 'voyage') === 0) {
	1304	+ $api_key = $options['voyage_api_key'] ?? '';
	1305	+ $provider = 'Voyage AI';
	1306	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	1307	+ $api_key = $options['gemini_api_key'] ?? '';
	1308	+ $provider = 'Google Gemini';
	1309	+ } else {
	1310	+ $api_key = $options['api_key'] ?? '';
	1311	+ $provider = 'OpenAI';
	1312	+ }
	1313	+
	1314	+ if (empty($api_key)) {
	1315	+ return array(
	1316	+ 'ok' => false,
	1317	+ 'api_key' => '',
	1318	+ 'reason' => sprintf(
	1319	+ /* translators: %s: embedding provider name */
	1320	+ __('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
	1321	+ $provider
	1322	+ ),
	1323	+ 'provider' => $provider,
	1324	+ );
	1325	+ }
	1326	+
	1327	+ return array('ok' => true, 'api_key' => $api_key, 'reason' => '', 'provider' => $provider);
	1328	+}
	1329	+
	1330	+/**
	1331	+ * Public QUERY-side entry point (plan 876edb). The chat pipeline's
	1332	+ * MxChat_Integrator::mxchat_generate_embedding() adapter routes through here
	1333	+ * so the query and index sides share ONE provider-routing implementation —
	1334	+ * the same endpoints, request bodies, and stamping semantics. The Integrator
	1335	+ * keeps its own error vocabulary by translating the WP_Error this returns
	1336	+ * (see the structured error data on every failure path below).
	1337	+ *
	1338	+ * @param string $text The text to be embedded.
	1339	+ * @param string $api_key Caller-resolved API key (per-bot on the query side).
	1340	+ * @param string $bot_id The bot ID for multi-bot support.
	1341	+ * @return array|WP_Error The embedding vector, or WP_Error carrying the reason.
	1342	+ */
	1343	+public static function generate_query_embedding($text, $api_key, $bot_id = 'default') {
	1344	+ return self::generate_embedding($text, $api_key, $bot_id);
	1345	+}
	1346	+
	1347	+/**
543	1348	* UPDATED: Generate an embedding for the given text using bot-specific configuration.
544	1349	*
545	1350	* @param string $text The text to be embedded.
546	1351	* @param string $api_key The API key used for generating embeddings.
		@@ -562,10 +1367,12 @@
562	1367	// below is untouched.
563	1368	if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
564	1369	$custom = self::generate_embedding_custom($text, $options);
565	1370	// The custom path already returns a human-readable error string —
566		- // carry it instead of collapsing to null (plan 4a7c0a).
567		- return is_array($custom) ? $custom : new WP_Error('embedding_failed', (string) $custom);
	1371	+ // carry it instead of collapsing to null (plan 4a7c0a). The 'custom'
	1372	+ // branch marker lets the Integrator adapter map the string back onto
	1373	+ // its own error codes (876edb).
	1374	+ return is_array($custom) ? $custom : new WP_Error('embedding_failed', (string) $custom, array('branch' => 'custom'));
568	1375	}
569	1376
570	1377	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
571	1378
		@@ -577,10 +1384,13 @@
577	1384	$endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
578	1385	$api_key = $options['gemini_api_key'] ?? '';
579	1386	} else {
580	1387	$endpoint = 'https://api.openai.com/v1/embeddings';
581		- // Use the bot-specific API key or fallback to passed API key
582		- $api_key = $options['api_key'] ?? $api_key;
	1388	+ // Prefer the caller-resolved key when one was passed — the query side
	1389	+ // resolves per-bot keys at its call sites (integrator adapter, 876edb).
	1390	+ // Index callers pass the preflight key, which equals this options read,
	1391	+ // so nothing changes for them.
	1392	+ $api_key = !empty($api_key) ? $api_key : ($options['api_key'] ?? '');
583	1393	}
584	1394
585	1395	// Prepare request body based on provider
586	1396	if (strpos($selected_model, 'gemini-embedding') === 0) {
		@@ -635,9 +1445,14 @@
635	1445	$message = 'Embedding request failed (connection): ' . $response->get_error_message();
636	1446	if (class_exists('MxChat_Admin')) {
637	1447	MxChat_Admin::mxchat_log_debug('embedding_error', $message, array('model' => $selected_model, 'bot_id' => $bot_id));
638	1448	}
639		- return new WP_Error('embedding_failed', $message);
	1449	+ return new WP_Error('embedding_failed', $message, array(
	1450	+ 'branch' => 'cloud',
	1451	+ 'kind' => 'connection',
	1452	+ 'reason' => $response->get_error_message(),
	1453	+ 'model' => $selected_model,
	1454	+ ));
640	1455	}
641	1456
642	1457	$response_body = json_decode(wp_remote_retrieve_body($response), true);
643	1458
		@@ -702,9 +1517,18 @@
702	1517	'bot_id' => $bot_id,
703	1518	));
704	1519	}
705	1520
706		- return new WP_Error('embedding_failed', $message);
	1521	+ // Structured data so the Integrator's query-side adapter can rebuild its
	1522	+ // typed error contract (auth/rate-limit/quota/invalid-response) without a
	1523	+ // second transport implementation (876edb). Additive — message unchanged.
	1524	+ return new WP_Error('embedding_failed', $message, array(
	1525	+ 'branch' => 'cloud',
	1526	+ 'status' => $status,
	1527	+ 'error_type' => (is_array($decoded) && isset($decoded['error']['type']) && is_string($decoded['error']['type'])) ? $decoded['error']['type'] : '',
	1528	+ 'reason' => $reason,
	1529	+ 'model' => $selected_model,
	1530	+ ));
707	1531	}
708	1532
709	1533	/**
710	1534	* Generate an embedding via a Custom (OpenAI-compatible) provider's /embeddings route.
		@@ -735,12 +1559,11 @@
735	1559	$api_key = isset($options['custom_provider_api_key']) ? trim((string) $options['custom_provider_api_key']) : '';
736	1560	$auth_scheme = isset($options['custom_provider_auth_scheme']) ? $options['custom_provider_auth_scheme'] : 'bearer';
737	1561	$api_version = isset($options['custom_provider_api_version']) ? trim((string) $options['custom_provider_api_version']) : '';
738	1562
739		- // Embedding model: prefer the dedicated custom_provider_embedding_model, fall back to the chat model.
740		- $model = (isset($options['custom_provider_embedding_model']) && trim((string) $options['custom_provider_embedding_model']) !== '')
741		- ? trim((string) $options['custom_provider_embedding_model'])
742		- : ((isset($options['custom_provider_model']) && trim((string) $options['custom_provider_model']) !== '') ? trim((string) $options['custom_provider_model']) : 'default');
	1563	+ // Embedding model: shared resolver (dedicated embedding model -> chat model
	1564	+ // -> 'default') — the mismatch warning's "selected" side reads the same chain.
	1565	+ $model = self::resolve_custom_embedding_model($options);
743	1566
744	1567	$embed_url = $base_url . '/embeddings';
745	1568	if (!empty($api_version)) {
746	1569	$embed_url .= (strpos($embed_url, '?') === false ? '?' : '&') . 'api-version=' . rawurlencode($api_version);
		@@ -760,9 +1583,13 @@
760	1583	'body' => wp_json_encode(['input' => $text, 'model' => $model]),
761	1584	'timeout' => 60,
762	1585	]);
763	1586	if (is_wp_error($response)) {
764		- return 'Connection error when generating embeddings (custom provider): ' . $response->get_error_message();
	1587	+ return self::log_custom_embedding_failure(
	1588	+ 'Connection error when generating embeddings (custom provider): ' . $response->get_error_message(),
	1589	+ $model,
	1590	+ $api_key
	1591	+ );
765	1592	}
766	1593
767	1594	$status = wp_remote_retrieve_response_code($response);
768	1595	$body = json_decode(wp_remote_retrieve_body($response), true);
		@@ -767,9 +1594,14 @@
767	1594	$status = wp_remote_retrieve_response_code($response);
768	1595	$body = json_decode(wp_remote_retrieve_body($response), true);
769	1596	if ($status !== 200) {
770	1597	$msg = isset($body['error']['message']) ? $body['error']['message'] : 'HTTP ' . $status;
771		- return 'Custom embedding endpoint error: ' . $msg;
	1598	+ return self::log_custom_embedding_failure(
	1599	+ 'Custom embedding endpoint error: ' . $msg,
	1600	+ $model,
	1601	+ $api_key,
	1602	+ (int) $status
	1603	+ );
772	1604	}
773	1605	if (isset($body['data'][0]['embedding']) && is_array($body['data'][0]['embedding'])) {
774	1606	// Stamp the custom model identity so the active-embedding-model mismatch
775	1607	// warning reflects the real (custom) model rather than the built-in setting.
		@@ -775,12 +1607,42 @@
775	1607	// warning reflects the real (custom) model rather than the built-in setting.
776	1608	self::stamp_active_embedding_model('custom:' . $model);
777	1609	return $body['data'][0]['embedding'];
778	1610	}
779		- return 'Invalid embedding response from custom provider.';
	1611	+ return self::log_custom_embedding_failure('Invalid embedding response from custom provider.', $model, $api_key);
780	1612	}
781	1613
782	1614	/**
	1615	+ * Record a custom-provider embedding failure in the Debug Mode log, then
	1616	+ * return the message unchanged so callers keep their string-error contract.
	1617	+ * The cloud branch has logged its failures since 4a7c0a; the custom branch
	1618	+ * never did, so chat-side failures on Custom-provider installs were
	1619	+ * invisible to Debug Mode despite the 3.2.18 readme saying otherwise
	1620	+ * (plan 71e4b6). Same scrub-then-log shape as embedding_failure_error().
	1621	+ *
	1622	+ * @param string $message Human-readable failure (the caller's return value).
	1623	+ * @param string $model Resolved custom embedding model.
	1624	+ * @param string $api_key Scrubbed out of the logged message if it ever appears.
	1625	+ * @param int $status HTTP status when one was received, 0 otherwise.
	1626	+ * @return string The (scrubbed) message.
	1627	+ */
	1628	+private static function log_custom_embedding_failure($message, $model, $api_key, $status = 0) {
	1629	+ if (is_string($api_key) && $api_key !== '') {
	1630	+ $message = str_replace($api_key, '[redacted]', $message);
	1631	+ }
	1632	+
	1633	+ if (class_exists('MxChat_Admin')) {
	1634	+ $context = array('model' => 'custom:' . $model);
	1635	+ if ($status > 0) {
	1636	+ $context['status'] = $status;
	1637	+ }
	1638	+ MxChat_Admin::mxchat_log_debug('embedding_error', $message, $context);
	1639	+ }
	1640	+
	1641	+ return $message;
	1642	+}
	1643	+
	1644	+/**
783	1645	* Submit content as multiple chunks
784	1646	*
785	1647	* Splits large content into chunks, generates embeddings for each,
786	1648	* and stores them with chunk metadata for later reassembly.
		@@ -799,10 +1661,26 @@
799	1661
800	1662	//error_log('[MXCHAT-CHUNK-DEBUG] Starting chunked submission for: ' . $source_url);
801	1663	//error_log('[MXCHAT-CHUNK-DEBUG] Content length: ' . strlen($content) . ' chars');
802	1664
803		- // First, delete any existing chunks for this URL (clean slate)
	1665	+ // URL-less (manual) content needs a minted identity BEFORE chunk ids are derived:
	1666	+ // every chunk id is md5(source_url)_chunk_N, so with source_url = '' EVERY long
	1667	+ // manual document shared the md5('') prefix — and the clean-slate delete below
	1668	+ // wiped the PREVIOUS manual entry's chunks each time a new one was added. The
	1669	+ // single-vector paths already mint (mxchat:// in WP, manual_* in Pinecone); this
	1670	+ // was the one storage path that didn't. Keep an mxchat:// identity if the caller
	1671	+ // already carries one.
	1672	+ if (strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, 'upload://') !== 0 && !preg_match('#^https?://#i', $source_url)) {
	1673	+ $source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
	1674	+ }
	1675	+
	1676	+ // First, delete any existing chunks for this URL (clean slate).
	1677	+ // Mirror-suspended: this is a re-store, not an entry removal — the Vector
	1678	+ // Store file is replaced (or kept, on hash match) by the caller's
	1679	+ // sync_upsert_entry after storage succeeds.
	1680	+ self::$vectorstore_mirror_suspended = true;
804	1681	$delete_result = self::delete_chunks_for_url($source_url, $bot_id);
	1682	+ self::$vectorstore_mirror_suspended = false;
805	1683	if (is_wp_error($delete_result)) {
806	1684	//error_log('[MXCHAT-CHUNK-DEBUG] Warning: Failed to delete existing chunks: ' . $delete_result->get_error_message());
807	1685	// Continue anyway - we'll overwrite with upsert
808	1686	}
		@@ -991,9 +1869,22 @@
991	1869	'last_updated' => time(),
992	1870	'created_at' => time(),
993	1871	'bot_id' => $bot_id,
994	1872	);
	1873	+ $page_title = self::source_title_for($source_url);
	1874	+ if ($page_title !== '') {
	1875	+ $metadata['title'] = $page_title; // page title captured at import (plan d9ee66)
	1876	+ }
995	1877
	1878	+ // Document index (plan 362c31): same chunk id and metadata, documents API.
	1879	+ if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) {
	1880	+ return MxChat_Pinecone_Documents::upsert_one($vector_id, $chunk_text, $embedding_vector, $metadata, array(
	1881	+ 'api_key' => $api_key,
	1882	+ 'host' => $host,
	1883	+ 'namespace' => $namespace,
	1884	+ ));
	1885	+ }
	1886	+
996	1887	$vector_data = array(
997	1888	'id' => $vector_id,
998	1889	'values' => $embedding_vector,
999	1890	'metadata' => $metadata
		@@ -1036,20 +1927,25 @@
1036	1927	global $wpdb;
1037	1928
1038	1929	// For chunks, we always insert new rows (no duplicate checking)
1039	1930	// The URL includes chunk info in the metadata, but source_url stays the same for grouping
1040		- $result = $wpdb->insert(
1041		- $table_name,
1042		- array(
1043		- 'url' => $source_url,
1044		- 'article_content' => $content_with_metadata,
1045		- 'embedding_vector' => $embedding_vector_serialized,
1046		- 'source_url' => $source_url,
1047		- 'content_type' => $content_type,
1048		- 'timestamp' => current_time('mysql')
1049		- ),
1050		- array('%s', '%s', '%s', '%s', '%s', '%s')
	1931	+ $row = array(
	1932	+ 'url' => $source_url,
	1933	+ 'article_content' => $content_with_metadata,
	1934	+ 'embedding_vector' => $embedding_vector_serialized,
	1935	+ 'source_url' => $source_url,
	1936	+ 'content_type' => $content_type,
	1937	+ 'timestamp' => current_time('mysql')
1051	1938	);
	1939	+ $formats = array('%s', '%s', '%s', '%s', '%s', '%s');
	1940	+ // Every chunk row carries the page title too (plan d9ee66) - readers group by
	1941	+ // source_url and may pick any chunk as the representative.
	1942	+ $page_title = self::source_title_for($source_url);
	1943	+ if ($page_title !== '' && self::ensure_kb_title_column()) {
	1944	+ $row['title'] = $page_title;
	1945	+ $formats[] = '%s';
	1946	+ }
	1947	+ $result = $wpdb->insert($table_name, $row, $formats);
1052	1948
1053	1949	if ($result === false) {
1054	1950	return new WP_Error('database_failed', 'Failed to insert chunk: ' . $wpdb->last_error);
1055	1951	}
		@@ -1057,8 +1953,142 @@
1057	1953	return true;
1058	1954	}
1059	1955
1060	1956	/**
	1957	+ * Option holding the page an uploaded file should be cited as (plan 025ea2).
	1958	+ * Keyed "<bot_id>|upload://<filename>" => https URL. The entry itself keeps
	1959	+ * its upload:// identity, so a re-upload of the same file still replaces the
	1960	+ * same entry and never collides with the cited page's own knowledge entry.
	1961	+ */
	1962	+const UPLOAD_SOURCE_URLS_OPTION = 'mxchat_upload_source_urls';
	1963	+
	1964	+/**
	1965	+ * Whether a stored source is an uploaded file (Document Upload / PDF Upload).
	1966	+ *
	1967	+ * @param string $source_url Stored source identity.
	1968	+ * @return bool
	1969	+ */
	1970	+public static function is_upload_source($source_url) {
	1971	+ return is_string($source_url) && stripos($source_url, 'upload://') === 0;
	1972	+}
	1973	+
	1974	+/**
	1975	+ * File name of an uploaded source, for a plain-text label.
	1976	+ *
	1977	+ * @param string $source_url Stored source identity (upload://<filename>).
	1978	+ * @return string
	1979	+ */
	1980	+public static function upload_source_name($source_url) {
	1981	+ return self::is_upload_source($source_url) ? substr(self::upload_source_base($source_url), strlen('upload://')) : '';
	1982	+}
	1983	+
	1984	+/**
	1985	+ * An uploaded source without its page fragment (plan dcc19d). A page of an
	1986	+ * uploaded PDF is stored as upload://<filename>#page=<n>; the file it belongs
	1987	+ * to, and the key its Source URL is kept under, is upload://<filename>.
	1988	+ *
	1989	+ * @param string $source_url Stored source identity.
	1990	+ * @return string
	1991	+ */
	1992	+public static function upload_source_base($source_url) {
	1993	+ return is_string($source_url) ? preg_replace('/#page=\d+$/', '', $source_url) : '';
	1994	+}
	1995	+
	1996	+/**
	1997	+ * Page number carried by an uploaded PDF page's identity, 0 when it has none.
	1998	+ *
	1999	+ * @param string $source_url Stored source identity.
	2000	+ * @return int
	2001	+ */
	2002	+public static function upload_source_page($source_url) {
	2003	+ return (self::is_upload_source($source_url) && preg_match('/#page=(\d+)$/', $source_url, $m)) ? (int) $m[1] : 0;
	2004	+}
	2005	+
	2006	+/**
	2007	+ * Identity one page of a PDF is stored under: <pdf>#page=<n>. A web PDF goes
	2008	+ * through esc_url() as before. An uploaded PDF (upload://<filename>) must not:
	2009	+ * esc_url() empties that scheme, which stored every page of an uploaded PDF
	2010	+ * as anonymous manual content, so the file could not be named, cited, given
	2011	+ * a Source URL, or replaced by a re-upload (plan dcc19d).
	2012	+ *
	2013	+ * @param string $pdf_url The PDF's own source (URL or upload://<filename>).
	2014	+ * @param int $page_number 1-based page number.
	2015	+ * @return string
	2016	+ */
	2017	+public static function pdf_page_identity($pdf_url, $page_number) {
	2018	+ $identity = $pdf_url . '#page=' . (int) $page_number;
	2019	+ return self::is_upload_source($pdf_url) ? sanitize_text_field($identity) : esc_url($identity);
	2020	+}
	2021	+
	2022	+/**
	2023	+ * Remember (or clear) the page an uploaded file should be cited as.
	2024	+ * Only http(s) URLs are kept; anything else clears the entry.
	2025	+ *
	2026	+ * @param string $source_label upload://<filename>
	2027	+ * @param string $bot_id Bot the upload belongs to.
	2028	+ * @param string $url Page URL, or '' to clear.
	2029	+ * @return string The URL stored, '' when cleared.
	2030	+ */
	2031	+public static function set_upload_source_url($source_label, $bot_id, $url) {
	2032	+ if (!self::is_upload_source($source_label)) {
	2033	+ return '';
	2034	+ }
	2035	+ $bot_id = ($bot_id === '' || $bot_id === null) ? 'default' : (string) $bot_id;
	2036	+ $url = is_string($url) ? esc_url_raw(trim($url), array('http', 'https')) : '';
	2037	+ if ($url !== '' && !preg_match('#^https?://#i', $url)) {
	2038	+ $url = '';
	2039	+ }
	2040	+
	2041	+ $map = get_option(self::UPLOAD_SOURCE_URLS_OPTION, array());
	2042	+ if (!is_array($map)) {
	2043	+ $map = array();
	2044	+ }
	2045	+ $key = $bot_id . '|' . $source_label;
	2046	+ if ($url === '') {
	2047	+ if (!isset($map[$key])) {
	2048	+ return '';
	2049	+ }
	2050	+ unset($map[$key]);
	2051	+ } else {
	2052	+ $map[$key] = $url;
	2053	+ }
	2054	+ if (empty($map)) {
	2055	+ delete_option(self::UPLOAD_SOURCE_URLS_OPTION);
	2056	+ } else {
	2057	+ update_option(self::UPLOAD_SOURCE_URLS_OPTION, $map, false);
	2058	+ }
	2059	+ return $url;
	2060	+}
	2061	+
	2062	+/**
	2063	+ * The page an uploaded file is cited as, '' when it has none.
	2064	+ * Looks under the asking bot first, then under the default bot (on the
	2065	+ * WordPress database store every bot reads the same entries).
	2066	+ *
	2067	+ * @param string $source_label upload://<filename>
	2068	+ * @param string $bot_id Bot answering the turn.
	2069	+ * @return string
	2070	+ */
	2071	+public static function get_upload_source_url($source_label, $bot_id = 'default') {
	2072	+ if (!self::is_upload_source($source_label)) {
	2073	+ return '';
	2074	+ }
	2075	+ $map = get_option(self::UPLOAD_SOURCE_URLS_OPTION, array());
	2076	+ if (!is_array($map) || empty($map)) {
	2077	+ return '';
	2078	+ }
	2079	+ // A PDF page (upload://<file>#page=<n>) is cited as its file's page.
	2080	+ $source_label = self::upload_source_base($source_label);
	2081	+ $bot_id = ($bot_id === '' || $bot_id === null) ? 'default' : (string) $bot_id;
	2082	+ foreach (array($bot_id . '|' . $source_label, 'default|' . $source_label) as $key) {
	2083	+ if (!empty($map[$key]) && preg_match('#^https?://#i', $map[$key])) {
	2084	+ return $map[$key];
	2085	+ }
	2086	+ }
	2087	+ return '';
	2088	+}
	2089	+
	2090	+/**
1061	2091	* Delete all chunks for a given URL
1062	2092	*
1063	2093	* @param string $source_url The source URL
1064	2094	* @param string $bot_id The bot ID
		@@ -1066,20 +2096,144 @@
1066	2096	*/
1067	2097	public static function delete_chunks_for_url($source_url, $bot_id = 'default') {
1068	2098	//error_log('[MXCHAT-CHUNK-DELETE] Deleting chunks for URL: ' . $source_url);
1069	2099
	2100	+ // Entry removal — mirror it to the Vector Store (unless a storage routine
	2101	+ // is mid-re-store, see $vectorstore_mirror_suspended).
	2102	+ if (!self::$vectorstore_mirror_suspended && class_exists('MxChat_Vectorstore_Manager')) {
	2103	+ MxChat_Vectorstore_Manager::sync_delete_entry($source_url, $bot_id);
	2104	+ }
	2105	+
1070	2106	if (self::is_pinecone_enabled_for_bot($bot_id)) {
1071		- return self::delete_pinecone_chunks_by_url($source_url, $bot_id);
	2107	+ $result = self::delete_pinecone_chunks_by_url($source_url, $bot_id);
1072	2108	} else {
1073		- return self::delete_wordpress_chunks_by_url($source_url);
	2109	+ $result = self::delete_wordpress_chunks_by_url($source_url);
1074	2110	}
	2111	+
	2112	+ // The entry is gone for good (not the clean-slate half of a re-store):
	2113	+ // a media file's "already indexed, nothing changed" signature must go
	2114	+ // with it, or auto-sync keeps skipping the file on every later save.
	2115	+ if (!self::$vectorstore_mirror_suspended && !is_wp_error($result)) {
	2116	+ self::forget_attachment_signature($source_url, $bot_id);
	2117	+ }
	2118	+
	2119	+ return $result;
1075	2120	}
1076	2121
1077	2122	/**
	2123	+ * Clear the media auto-sync signature (_mxchat_kb_attachment_sig) of the
	2124	+ * attachment a just-deleted knowledge entry was indexed from.
	2125	+ *
	2126	+ * Call it wherever a WHOLE entry is removed (never for the clean-slate half
	2127	+ * of a re-store, and never when sibling chunk rows survive). Posts and pages
	2128	+ * carry no signature; for them this is two indexed lookups that find nothing.
	2129	+ *
	2130	+ * @param string $source_url The entry's source URL.
	2131	+ * @param string $bot_id The bot whose entry was removed.
	2132	+ */
	2133	+public static function forget_attachment_signature($source_url, $bot_id = 'default') {
	2134	+ global $wpdb;
	2135	+
	2136	+ $source_url = (string) $source_url;
	2137	+ if (!preg_match('#^https?://#i', $source_url) || !self::attachment_signature_follows_bot($bot_id)) {
	2138	+ return;
	2139	+ }
	2140	+
	2141	+ $attachment_ids = array();
	2142	+ $by_url = attachment_url_to_postid($source_url);
	2143	+ if ($by_url) {
	2144	+ $attachment_ids[] = (int) $by_url;
	2145	+ }
	2146	+
	2147	+ // The signature stores the exact URL the indexer used, which can differ
	2148	+ // from where the file lives now (replaced file, scaled variant). Bounded
	2149	+ // by the number of signed attachments: the meta_key index does the work.
	2150	+ $candidates = $wpdb->get_results($wpdb->prepare(
	2151	+ "SELECT post_id, meta_value FROM {$wpdb->postmeta}
	2152	+ WHERE meta_key = '_mxchat_kb_attachment_sig' AND meta_value LIKE %s
	2153	+ LIMIT 20",
	2154	+ '%' . $wpdb->esc_like($source_url) . '%'
	2155	+ ));
	2156	+ foreach ((array) $candidates as $candidate) {
	2157	+ $signature = maybe_unserialize($candidate->meta_value);
	2158	+ if (is_array($signature) && isset($signature['url']) && $signature['url'] === $source_url) {
	2159	+ $attachment_ids[] = (int) $candidate->post_id;
	2160	+ }
	2161	+ }
	2162	+
	2163	+ foreach (array_unique($attachment_ids) as $attachment_id) {
	2164	+ delete_post_meta($attachment_id, '_mxchat_kb_attachment_sig');
	2165	+ }
	2166	+}
	2167	+
	2168	+/**
	2169	+ * The same, for callers that hold a Pinecone vector id instead of a URL.
	2170	+ * Only a base id (md5 of the source URL) names a whole entry; a single
	2171	+ * chunk id leaves the rest of the entry in place, so it is ignored.
	2172	+ */
	2173	+public static function forget_attachment_signature_by_key($entry_key, $bot_id = 'default') {
	2174	+ global $wpdb;
	2175	+
	2176	+ $entry_key = (string) $entry_key;
	2177	+ if (!preg_match('/^[a-f0-9]{32}$/', $entry_key) || !self::attachment_signature_follows_bot($bot_id)) {
	2178	+ return;
	2179	+ }
	2180	+
	2181	+ $signed = $wpdb->get_results(
	2182	+ "SELECT post_id, meta_value FROM {$wpdb->postmeta}
	2183	+ WHERE meta_key = '_mxchat_kb_attachment_sig' LIMIT 5000"
	2184	+ );
	2185	+ foreach ((array) $signed as $row) {
	2186	+ $signature = maybe_unserialize($row->meta_value);
	2187	+ if (is_array($signature) && isset($signature['url']) && md5((string) $signature['url']) === $entry_key) {
	2188	+ delete_post_meta((int) $row->post_id, '_mxchat_kb_attachment_sig');
	2189	+ }
	2190	+ }
	2191	+}
	2192	+
	2193	+/**
	2194	+ * Every media entry of a bot was removed at once ("Delete all").
	2195	+ */
	2196	+public static function forget_all_attachment_signatures($bot_id = 'default') {
	2197	+ if ($bot_id !== 'default' && $bot_id !== 'testing' && !self::is_pinecone_enabled_for_bot($bot_id)) {
	2198	+ // WordPress-database "Delete all" for another bot removes that bot's
	2199	+ // rows only; the default bot's entries are still there.
	2200	+ return;
	2201	+ }
	2202	+ if (self::attachment_signature_follows_bot($bot_id)) {
	2203	+ delete_post_meta_by_key('_mxchat_kb_attachment_sig');
	2204	+ }
	2205	+}
	2206	+
	2207	+/**
	2208	+ * The signature vouches for the DEFAULT bot's entry (media auto-sync only
	2209	+ * indexes for the default bot). Removing another bot's copy from a Pinecone
	2210	+ * store of its own leaves that entry, and so the signature, in place.
	2211	+ */
	2212	+private static function attachment_signature_follows_bot($bot_id) {
	2213	+ if ($bot_id === 'default' || $bot_id === 'testing' || !self::is_pinecone_enabled_for_bot($bot_id)) {
	2214	+ return true;
	2215	+ }
	2216	+ if (!class_exists('MxChat_Pinecone_Documents') || !self::is_pinecone_enabled_for_bot('default')) {
	2217	+ return false;
	2218	+ }
	2219	+ $theirs = MxChat_Pinecone_Documents::config($bot_id);
	2220	+ $ours = MxChat_Pinecone_Documents::config('default');
	2221	+ return strtolower((string) $theirs['host']) === strtolower((string) $ours['host'])
	2222	+ && (string) $theirs['namespace'] === (string) $ours['namespace'];
	2223	+}
	2224	+
	2225	+/**
1078	2226	* Delete all chunks for a URL from Pinecone
1079	2227	*/
1080		-private static function delete_pinecone_chunks_by_url($source_url, $bot_id) {
1081		- // Get Pinecone configuration
	2228	+/**
	2229	+ * Delete leftover md5(url)_chunk_N vectors after a URL's content was re-stored
	2230	+ * as a SINGLE vector (content shrank below the chunk threshold on edit/re-import).
	2231	+ * Unlike delete_pinecone_chunks_by_url this leaves the base id alone — the caller
	2232	+ * just upserted the new content there. Failure is logged, not fatal: the save
	2233	+ * itself succeeded, and the next save retries the sweep.
	2234	+ */
	2235	+private static function cleanup_pinecone_chunk_stragglers($source_url, $bot_id) {
1082	2236	if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
1083	2237	$pinecone_options = get_option('mxchat_pinecone_addon_options');
1084	2238	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
1085	2239	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
		@@ -1098,21 +2252,30 @@
1098	2252	}
1099	2253	}
1100	2254
1101	2255	if (empty($host) || empty($api_key)) {
1102		- return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
	2256	+ return;
1103	2257	}
1104	2258
1105		- $base_vector_id = md5($source_url);
1106		- $vectors_to_delete = array();
	2259	+ // Document index (plan 362c31): list the chunk ids and delete them through the documents API.
	2260	+ if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) {
	2261	+ $docs_cfg = array('api_key' => $api_key, 'host' => $host, 'namespace' => $namespace);
	2262	+ $doc_stragglers = MxChat_Pinecone_Documents::list_ids(md5($source_url) . '_chunk_', $docs_cfg);
	2263	+ if (!empty($doc_stragglers)) {
	2264	+ $doc_result = MxChat_Pinecone_Documents::delete_ids($doc_stragglers, $docs_cfg);
	2265	+ if (is_wp_error($doc_result) && class_exists('MxChat_Admin') && method_exists('MxChat_Admin', 'mxchat_log_debug')) {
	2266	+ MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to sweep stale chunk documents after single-vector re-store', array('source_url' => $source_url, 'bot_id' => $bot_id, 'count' => count($doc_stragglers)));
	2267	+ }
	2268	+ }
	2269	+ return;
	2270	+ }
1107	2271
1108		- // Add the original single-vector ID (for non-chunked content)
1109		- $vectors_to_delete[] = $base_vector_id;
	2272	+ $stragglers = array();
1110	2273
1111		- // Pinecone /vectors/list is a GET endpoint with query-string parameters; a POST here returns a
1112		- // non-200 silently and we end up only deleting the base vector, leaving chunks orphaned.
	2274	+ // Pinecone /vectors/list is a GET endpoint with query-string parameters (a POST
	2275	+ // answers 200-with-an-empty-body, which reads as "no stragglers").
1113	2276	$query_params = array(
1114		- 'prefix' => $base_vector_id . '_chunk_',
	2277	+ 'prefix' => md5($source_url) . '_chunk_',
1115	2278	'limit' => 100,
1116	2279	);
1117	2280	if (!empty($namespace)) {
1118	2281	$query_params['namespace'] = $namespace;
		@@ -1119,9 +2282,8 @@
1119	2282	}
1120	2283
1121	2284	$list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
1122	2285
1123		- // Paginate in case a URL has more than 100 chunks.
1124	2286	do {
1125	2287	$list_response = wp_remote_get($list_url, array(
1126	2288	'headers' => array(
1127	2289	'Api-Key' => $api_key,
		@@ -1137,9 +2299,9 @@
1137	2299	$list_data = json_decode(wp_remote_retrieve_body($list_response), true);
1138	2300	if (!empty($list_data['vectors'])) {
1139	2301	foreach ($list_data['vectors'] as $vector) {
1140	2302	if (isset($vector['id'])) {
1141		- $vectors_to_delete[] = $vector['id'];
	2303	+ $stragglers[] = $vector['id'];
1142	2304	}
1143	2305	}
1144	2306	}
1145	2307
		@@ -1151,8 +2313,188 @@
1151	2313	$query_params['paginationToken'] = $next_token;
1152	2314	$list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
1153	2315	} while (true);
1154	2316
	2317	+ if (empty($stragglers)) {
	2318	+ return;
	2319	+ }
	2320	+
	2321	+ $delete_body = array('ids' => $stragglers);
	2322	+ if (!empty($namespace)) {
	2323	+ $delete_body['namespace'] = $namespace;
	2324	+ }
	2325	+
	2326	+ $delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
	2327	+ 'headers' => array(
	2328	+ 'Api-Key' => $api_key,
	2329	+ 'accept' => 'application/json',
	2330	+ 'content-type' => 'application/json'
	2331	+ ),
	2332	+ 'body' => wp_json_encode($delete_body),
	2333	+ 'timeout' => 30
	2334	+ ));
	2335	+
	2336	+ if ((is_wp_error($delete_response) || wp_remote_retrieve_response_code($delete_response) !== 200)
	2337	+ && class_exists('MxChat_Admin') && method_exists('MxChat_Admin', 'mxchat_log_debug')) {
	2338	+ MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to sweep stale chunk vectors after single-vector re-store', array('source_url' => $source_url, 'bot_id' => $bot_id, 'count' => count($stragglers)));
	2339	+ }
	2340	+}
	2341	+
	2342	+/**
	2343	+ * Resolve the Pinecone credentials for a bot: the bot's own config when the
	2344	+ * multi-bot add-on supplies one, else the site-wide options. Extracted so the
	2345	+ * delete path and the count path can never resolve different credentials.
	2346	+ *
	2347	+ * @param string $bot_id
	2348	+ * @return array{api_key:string,host:string,namespace:string}
	2349	+ */
	2350	+private static function pinecone_config_for_bot($bot_id) {
	2351	+ if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
	2352	+ $bot_pinecone_config = array();
	2353	+ } else {
	2354	+ $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
	2355	+ }
	2356	+
	2357	+ if (empty($bot_pinecone_config)) {
	2358	+ $pinecone_options = get_option('mxchat_pinecone_addon_options');
	2359	+ return array(
	2360	+ 'api_key' => $pinecone_options['mxchat_pinecone_api_key'] ?? '',
	2361	+ 'host' => $pinecone_options['mxchat_pinecone_host'] ?? '',
	2362	+ 'namespace' => $pinecone_options['mxchat_pinecone_namespace'] ?? '',
	2363	+ // 'vector' | 'document' (plan 362c31) — the delete / count paths branch on it.
	2364	+ 'index_type' => (($pinecone_options['mxchat_pinecone_index_type'] ?? 'vector') === 'document') ? 'document' : 'vector',
	2365	+ );
	2366	+ }
	2367	+
	2368	+ return array(
	2369	+ 'api_key' => $bot_pinecone_config['api_key'] ?? '',
	2370	+ 'host' => $bot_pinecone_config['host'] ?? '',
	2371	+ 'namespace' => $bot_pinecone_config['namespace'] ?? '',
	2372	+ 'index_type' => (($bot_pinecone_config['index_type'] ?? 'vector') === 'document') ? 'document' : 'vector',
	2373	+ );
	2374	+}
	2375	+
	2376	+/**
	2377	+ * List every Pinecone vector id under a prefix, following pagination.
	2378	+ *
	2379	+ * Pinecone /vectors/list is a GET endpoint with query-string parameters; a POST here returns a
	2380	+ * non-200 silently and we end up seeing only the base vector, leaving chunks orphaned.
	2381	+ *
	2382	+ * @param string $prefix Vector-id prefix to match.
	2383	+ * @param array $cfg From pinecone_config_for_bot().
	2384	+ * @return array List of vector ids (empty when none match or the API is unreachable).
	2385	+ */
	2386	+private static function pinecone_list_ids_by_prefix($prefix, array $cfg) {
	2387	+ // Document index (plan 362c31): the documents API lists by prefix too.
	2388	+ if (($cfg['index_type'] ?? 'vector') === 'document' && class_exists('MxChat_Pinecone_Documents')) {
	2389	+ return MxChat_Pinecone_Documents::list_ids($prefix, $cfg);
	2390	+ }
	2391	+
	2392	+ $ids = array();
	2393	+
	2394	+ $query_params = array(
	2395	+ 'prefix' => $prefix,
	2396	+ 'limit' => 100,
	2397	+ );
	2398	+ if (!empty($cfg['namespace'])) {
	2399	+ $query_params['namespace'] = $cfg['namespace'];
	2400	+ }
	2401	+
	2402	+ $list_url = "https://{$cfg['host']}/vectors/list?" . http_build_query($query_params);
	2403	+
	2404	+ // Paginate in case a URL has more than 100 chunks.
	2405	+ do {
	2406	+ $list_response = wp_remote_get($list_url, array(
	2407	+ 'headers' => array(
	2408	+ 'Api-Key' => $cfg['api_key'],
	2409	+ 'accept' => 'application/json',
	2410	+ ),
	2411	+ 'timeout' => 30,
	2412	+ ));
	2413	+
	2414	+ if (is_wp_error($list_response) || wp_remote_retrieve_response_code($list_response) !== 200) {
	2415	+ break;
	2416	+ }
	2417	+
	2418	+ $list_data = json_decode(wp_remote_retrieve_body($list_response), true);
	2419	+ if (!empty($list_data['vectors'])) {
	2420	+ foreach ($list_data['vectors'] as $vector) {
	2421	+ if (isset($vector['id'])) {
	2422	+ $ids[] = $vector['id'];
	2423	+ }
	2424	+ }
	2425	+ }
	2426	+
	2427	+ $next_token = $list_data['pagination']['next'] ?? '';
	2428	+ if (empty($next_token)) {
	2429	+ break;
	2430	+ }
	2431	+
	2432	+ $query_params['paginationToken'] = $next_token;
	2433	+ $list_url = "https://{$cfg['host']}/vectors/list?" . http_build_query($query_params);
	2434	+ } while (true);
	2435	+
	2436	+ return $ids;
	2437	+}
	2438	+
	2439	+/**
	2440	+ * How many stored pieces a URL currently has, in whichever backend holds it.
	2441	+ *
	2442	+ * Branches on the SAME condition delete_chunks_for_url() branches on, so the
	2443	+ * number reported is the number that call removes. Used by the REST DELETE
	2444	+ * route to answer "was anything actually there?" without a second delete path.
	2445	+ *
	2446	+ * @param string $source_url
	2447	+ * @param string $bot_id
	2448	+ * @return int
	2449	+ */
	2450	+public static function count_chunks_for_url($source_url, $bot_id = 'default') {
	2451	+ if (self::is_pinecone_enabled_for_bot($bot_id)) {
	2452	+ $cfg = self::pinecone_config_for_bot($bot_id);
	2453	+ if (empty($cfg['host']) || empty($cfg['api_key'])) {
	2454	+ return 0;
	2455	+ }
	2456	+ // The base id and every "<base>_chunk_N" share the base as a prefix,
	2457	+ // so one prefix listing covers chunked and non-chunked content alike.
	2458	+ return count(self::pinecone_list_ids_by_prefix(md5($source_url), $cfg));
	2459	+ }
	2460	+
	2461	+ global $wpdb;
	2462	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	2463	+
	2464	+ return (int) $wpdb->get_var($wpdb->prepare(
	2465	+ "SELECT COUNT(*) FROM {$table_name} WHERE source_url = %s",
	2466	+ $source_url
	2467	+ ));
	2468	+}
	2469	+
	2470	+private static function delete_pinecone_chunks_by_url($source_url, $bot_id) {
	2471	+ $cfg = self::pinecone_config_for_bot($bot_id);
	2472	+ $api_key = $cfg['api_key'];
	2473	+ $host = $cfg['host'];
	2474	+ $namespace = $cfg['namespace'];
	2475	+
	2476	+ if (empty($host) || empty($api_key)) {
	2477	+ return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
	2478	+ }
	2479	+
	2480	+ // Document index (plan 362c31): base id + every <md5>_chunk_N, documents API.
	2481	+ if (($cfg['index_type'] ?? 'vector') === 'document' && class_exists('MxChat_Pinecone_Documents')) {
	2482	+ $doc_ids = array_merge(array(md5($source_url)), MxChat_Pinecone_Documents::list_ids(md5($source_url) . '_chunk_', $cfg));
	2483	+ return MxChat_Pinecone_Documents::delete_ids($doc_ids, $cfg);
	2484	+ }
	2485	+
	2486	+ $base_vector_id = md5($source_url);
	2487	+ $vectors_to_delete = array();
	2488	+
	2489	+ // Add the original single-vector ID (for non-chunked content)
	2490	+ $vectors_to_delete[] = $base_vector_id;
	2491	+
	2492	+ $vectors_to_delete = array_merge(
	2493	+ $vectors_to_delete,
	2494	+ self::pinecone_list_ids_by_prefix($base_vector_id . '_chunk_', $cfg)
	2495	+ );
	2496	+
1155	2497	if (empty($vectors_to_delete)) {
1156	2498	//error_log('[MXCHAT-CHUNK-DELETE] No vectors found to delete');
1157	2499	return true;
1158	2500	}
		@@ -1247,6 +2589,261 @@
1247	2589
1248	2590	$capability = $index_exists ? 'fulltext' : 'like';
1249	2591	update_option('mxchat_hybrid_keyword_capability', $capability);
1250	2592	return $capability;
	2593	+}
	2594	+
	2595	+/**
	2596	+ * Public entry for re-embedding already-stored content in place (wp mxchat
	2597	+ * rtl-repair, plan d1e6f7). Thin wrapper so the repair CLI gets the exact
	2598	+ * provider routing the import path uses — the repaired vector must come from
	2599	+ * the same model family the bot indexes with, or retrieval stays broken.
	2600	+ */
	2601	+public static function regenerate_embedding($text, $api_key, $bot_id = 'default') {
	2602	+ return self::generate_embedding($text, $api_key, $bot_id);
	2603	+}
	2604	+
	2605	+/**
	2606	+ * Restore logical character order in PDF-extracted RTL text (plan 32bf9e).
	2607	+ *
	2608	+ * The bundled Smalot parser only un-reverses text runs tagged with the
	2609	+ * ReversedChars marked-content operator (Word emits it; LibreOffice and most
	2610	+ * other producers do not), so their Hebrew/Arabic PDFs extract in visual
	2611	+ * (reversed) order and embed/search as garbage. This is OUR post-processing
	2612	+ * seam over getText() — the parser itself is never patched (it gets replaced
	2613	+ * wholesale on library updates).
	2614	+ *
	2615	+ * Heuristic and deliberately conservative, per line:
	2616	+ * - lines without strong RTL codepoints are untouched (a fully-Latin line in
	2617	+ * an RTL document therefore stays as extracted — accepted limitation);
	2618	+ * - Arabic presentation forms are a definitive visual-order signal (they only
	2619	+ * appear in shaped output): de-shape to base letters and reverse;
	2620	+ * - otherwise flip only on positive evidence — Hebrew final-letter position
	2621	+ * (a sofit at word START only happens in reversed text) or sentence
	2622	+ * punctuation position (leading in visual order, trailing in logical);
	2623	+ * - ambiguous lines are left alone: a conservative miss beats corrupting a
	2624	+ * Word-produced extraction the parser already handled (the double-flip
	2625	+ * guard this plan's approval named mandatory).
	2626	+ *
	2627	+ * @param string $text One extracted page string, straight from getText().
	2628	+ * @param string $context Caller tag for the Debug Mode entry (site + page).
	2629	+ * @return string Text with RTL lines restored to logical order.
	2630	+ */
	2631	+public static function normalize_pdf_rtl($text, $context = '') {
	2632	+ if (!is_string($text) || '' === $text) {
	2633	+ return $text;
	2634	+ }
	2635	+ // Escape hatch for sites whose PDFs already extract logically.
	2636	+ if (!apply_filters('mxchat_pdf_rtl_normalize', true, $text)) {
	2637	+ return $text;
	2638	+ }
	2639	+ // Fast bail: nothing RTL anywhere in the page.
	2640	+ if (!preg_match('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $text)) {
	2641	+ return $text;
	2642	+ }
	2643	+
	2644	+ $parts = preg_split('/(\R)/u', $text, -1, PREG_SPLIT_DELIM_CAPTURE);
	2645	+ if (false === $parts) {
	2646	+ return $text;
	2647	+ }
	2648	+
	2649	+ $flipped_lines = 0;
	2650	+ $deshaped_lines = 0;
	2651	+ foreach ($parts as $i => $part) {
	2652	+ if ('' === $part || preg_match('/^\R$/u', $part)) {
	2653	+ continue;
	2654	+ }
	2655	+ $was_flipped = false;
	2656	+ $was_deshaped = false;
	2657	+ $new = self::pdf_rtl_normalize_line($part, $was_flipped, $was_deshaped);
	2658	+ if ($new !== $part) {
	2659	+ $parts[$i] = $new;
	2660	+ }
	2661	+ if ($was_flipped) {
	2662	+ $flipped_lines++;
	2663	+ }
	2664	+ if ($was_deshaped) {
	2665	+ $deshaped_lines++;
	2666	+ }
	2667	+ }
	2668	+
	2669	+ if (($flipped_lines || $deshaped_lines) && class_exists('MxChat_Admin')) {
	2670	+ MxChat_Admin::mxchat_log_debug('pdf_rtl_normalized', 'RTL PDF text restored to logical order', array(
	2671	+ 'context' => (string) $context,
	2672	+ 'lines_flipped' => $flipped_lines,
	2673	+ 'lines_deshaped' => $deshaped_lines,
	2674	+ 'decision' => 'visual-order extraction detected',
	2675	+ ));
	2676	+ }
	2677	+
	2678	+ return implode('', $parts);
	2679	+}
	2680	+
	2681	+/**
	2682	+ * Normalize one line. Sets $flipped/$deshaped for the caller's debug entry.
	2683	+ */
	2684	+private static function pdf_rtl_normalize_line($line, &$flipped, &$deshaped) {
	2685	+ $flipped = false;
	2686	+ $deshaped = false;
	2687	+
	2688	+ if (!preg_match('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $line)) {
	2689	+ return $line;
	2690	+ }
	2691	+
	2692	+ $has_forms = (bool) preg_match('/[\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $line);
	2693	+ $work = $line;
	2694	+ if ($has_forms) {
	2695	+ $work = strtr($work, self::pdf_rtl_deshape_map());
	2696	+ $deshaped = ($work !== $line);
	2697	+ }
	2698	+
	2699	+ $verdict = 'ambiguous';
	2700	+ if ($has_forms) {
	2701	+ // Shaped glyph codepoints only exist in visual-order output.
	2702	+ $verdict = 'visual';
	2703	+ } else {
	2704	+ // Strong-direction dominance gate first: an LTR-dominant line with an
	2705	+ // embedded RTL word is not flip material.
	2706	+ $rtl_count = preg_match_all('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}]/u', $work, $m_rtl);
	2707	+ $ltr_count = preg_match_all('/[A-Za-z]/u', $work, $m_ltr);
	2708	+ if ($rtl_count < 1 || $rtl_count <= $ltr_count) {
	2709	+ return $line;
	2710	+ }
	2711	+
	2712	+ // Hebrew final letters (ך ם ן ף ץ) end words in logical text; one at
	2713	+ // a word START (Hebrew letter follows, none precedes) is reversal
	2714	+ // evidence. Positional, so it survives the line being reversed.
	2715	+ $sofit_initial = preg_match_all('/(?<![\x{05D0}-\x{05EA}])[\x{05DA}\x{05DD}\x{05DF}\x{05E3}\x{05E5}](?=[\x{05D0}-\x{05EA}])/u', $work, $m_i);
	2716	+ $sofit_terminal = preg_match_all('/(?<=[\x{05D0}-\x{05EA}])[\x{05DA}\x{05DD}\x{05DF}\x{05E3}\x{05E5}](?![\x{05D0}-\x{05EA}])/u', $work, $m_t);
	2717	+ if ($sofit_initial > $sofit_terminal) {
	2718	+ $verdict = 'visual';
	2719	+ } elseif ($sofit_terminal > $sofit_initial) {
	2720	+ $verdict = 'logical';
	2721	+ } else {
	2722	+ // Sentence punctuation lands at the visual LEFT edge of an RTL
	2723	+ // line, i.e. the START of a visual-order extraction.
	2724	+ $trimmed = trim($work);
	2725	+ $starts_punct = (bool) preg_match('/^[.?!:;,]/u', $trimmed);
	2726	+ $ends_punct = (bool) preg_match('/[.?!:;,]$/u', $trimmed);
	2727	+ if ($starts_punct && !$ends_punct) {
	2728	+ $verdict = 'visual';
	2729	+ } elseif ($ends_punct && !$starts_punct) {
	2730	+ $verdict = 'logical';
	2731	+ }
	2732	+ }
	2733	+ }
	2734	+
	2735	+ if ('visual' !== $verdict) {
	2736	+ // Ambiguous or logical: hand back the original line UNLESS we
	2737	+ // de-shaped (de-shaping alone is always safe — same letters, same
	2738	+ // order, un-ligated).
	2739	+ return $deshaped ? $work : $line;
	2740	+ }
	2741	+
	2742	+ $flipped = true;
	2743	+ return self::pdf_rtl_flip_line($work);
	2744	+}
	2745	+
	2746	+/**
	2747	+ * Reverse a visual-order line into logical order: full character reversal,
	2748	+ * mirror paired punctuation, then re-reverse embedded LTR runs (Latin words
	2749	+ * and digit sequences, incl. Arabic-Indic digits) so they stay readable.
	2750	+ */
	2751	+private static function pdf_rtl_flip_line($line) {
	2752	+ $chars = preg_split('//u', $line, -1, PREG_SPLIT_NO_EMPTY);
	2753	+ if (false === $chars) {
	2754	+ return $line;
	2755	+ }
	2756	+ $reversed = implode('', array_reverse($chars));
	2757	+ $reversed = strtr($reversed, array(
	2758	+ '(' => ')', ')' => '(',
	2759	+ '[' => ']', ']' => '[',
	2760	+ '{' => '}', '}' => '{',
	2761	+ '<' => '>', '>' => '<',
	2762	+ ));
	2763	+ $restored = preg_replace_callback(
	2764	+ '/[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9}](?:[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9} .,\'"%\-:\/]*[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9}])?/u',
	2765	+ function ($m) {
	2766	+ $run = preg_split('//u', $m[0], -1, PREG_SPLIT_NO_EMPTY);
	2767	+ return false === $run ? $m[0] : implode('', array_reverse($run));
	2768	+ },
	2769	+ $reversed
	2770	+ );
	2771	+ return null === $restored ? $reversed : $restored;
	2772	+}
	2773	+
	2774	+/**
	2775	+ * Arabic presentation forms (A + B) -> base letters. Built once from range
	2776	+ * specs rather than ~120 hand-written literal entries; every codepoint in a
	2777	+ * range maps to the same base sequence (isolated/final/initial/medial forms
	2778	+ * of one letter are contiguous in the FE70 block).
	2779	+ */
	2780	+private static function pdf_rtl_deshape_map() {
	2781	+ static $map = null;
	2782	+ if (null !== $map) {
	2783	+ return $map;
	2784	+ }
	2785	+ $ranges = array(
	2786	+ // Form B harakat (each pair = standalone + tatweel-joined form).
	2787	+ array(0xFE70, 0xFE71, array(0x064B)), array(0xFE72, 0xFE72, array(0x064C)),
	2788	+ array(0xFE74, 0xFE74, array(0x064D)), array(0xFE76, 0xFE77, array(0x064E)),
	2789	+ array(0xFE78, 0xFE79, array(0x064F)), array(0xFE7A, 0xFE7B, array(0x0650)),
	2790	+ array(0xFE7C, 0xFE7D, array(0x0651)), array(0xFE7E, 0xFE7F, array(0x0652)),
	2791	+ // Form B letters.
	2792	+ array(0xFE80, 0xFE80, array(0x0621)), array(0xFE81, 0xFE82, array(0x0622)),
	2793	+ array(0xFE83, 0xFE84, array(0x0623)), array(0xFE85, 0xFE86, array(0x0624)),
	2794	+ array(0xFE87, 0xFE88, array(0x0625)), array(0xFE89, 0xFE8C, array(0x0626)),
	2795	+ array(0xFE8D, 0xFE8E, array(0x0627)), array(0xFE8F, 0xFE92, array(0x0628)),
	2796	+ array(0xFE93, 0xFE94, array(0x0629)), array(0xFE95, 0xFE98, array(0x062A)),
	2797	+ array(0xFE99, 0xFE9C, array(0x062B)), array(0xFE9D, 0xFEA0, array(0x062C)),
	2798	+ array(0xFEA1, 0xFEA4, array(0x062D)), array(0xFEA5, 0xFEA8, array(0x062E)),
	2799	+ array(0xFEA9, 0xFEAA, array(0x062F)), array(0xFEAB, 0xFEAC, array(0x0630)),
	2800	+ array(0xFEAD, 0xFEAE, array(0x0631)), array(0xFEAF, 0xFEB0, array(0x0632)),
	2801	+ array(0xFEB1, 0xFEB4, array(0x0633)), array(0xFEB5, 0xFEB8, array(0x0634)),
	2802	+ array(0xFEB9, 0xFEBC, array(0x0635)), array(0xFEBD, 0xFEC0, array(0x0636)),
	2803	+ array(0xFEC1, 0xFEC4, array(0x0637)), array(0xFEC5, 0xFEC8, array(0x0638)),
	2804	+ array(0xFEC9, 0xFECC, array(0x0639)), array(0xFECD, 0xFED0, array(0x063A)),
	2805	+ array(0xFED1, 0xFED4, array(0x0641)), array(0xFED5, 0xFED8, array(0x0642)),
	2806	+ array(0xFED9, 0xFEDC, array(0x0643)), array(0xFEDD, 0xFEE0, array(0x0644)),
	2807	+ array(0xFEE1, 0xFEE4, array(0x0645)), array(0xFEE5, 0xFEE8, array(0x0646)),
	2808	+ array(0xFEE9, 0xFEEC, array(0x0647)), array(0xFEED, 0xFEEE, array(0x0648)),
	2809	+ array(0xFEEF, 0xFEF0, array(0x0649)), array(0xFEF1, 0xFEF4, array(0x064A)),
	2810	+ // Form B lam-alef ligatures decompose to two letters.
	2811	+ array(0xFEF5, 0xFEF6, array(0x0644, 0x0622)), array(0xFEF7, 0xFEF8, array(0x0644, 0x0623)),
	2812	+ array(0xFEF9, 0xFEFA, array(0x0644, 0x0625)), array(0xFEFB, 0xFEFC, array(0x0644, 0x0627)),
	2813	+ // Form A: Persian / Urdu letters in common use.
	2814	+ array(0xFB56, 0xFB59, array(0x067E)), array(0xFB66, 0xFB69, array(0x0679)),
	2815	+ array(0xFB7A, 0xFB7D, array(0x0686)), array(0xFB88, 0xFB89, array(0x0688)),
	2816	+ array(0xFB8A, 0xFB8B, array(0x0698)), array(0xFB8E, 0xFB91, array(0x06A9)),
	2817	+ array(0xFB92, 0xFB95, array(0x06AF)), array(0xFBA6, 0xFBA9, array(0x06C1)),
	2818	+ array(0xFBAA, 0xFBAD, array(0x06BE)), array(0xFBAE, 0xFBAF, array(0x06D2)),
	2819	+ array(0xFBFC, 0xFBFF, array(0x06CC)),
	2820	+ );
	2821	+ $map = array();
	2822	+ foreach ($ranges as $range) {
	2823	+ $base = '';
	2824	+ foreach ($range[2] as $cp) {
	2825	+ $base .= self::pdf_rtl_cp_to_utf8($cp);
	2826	+ }
	2827	+ for ($cp = $range[0]; $cp <= $range[1]; $cp++) {
	2828	+ $map[self::pdf_rtl_cp_to_utf8($cp)] = $base;
	2829	+ }
	2830	+ }
	2831	+ return $map;
	2832	+}
	2833	+
	2834	+/**
	2835	+ * Codepoint to UTF-8 without ext-intl / mbstring entity tricks (PHP 7.2 floor).
	2836	+ */
	2837	+private static function pdf_rtl_cp_to_utf8($cp) {
	2838	+ if ($cp < 0x80) {
	2839	+ return chr($cp);
	2840	+ }
	2841	+ if ($cp < 0x800) {
	2842	+ return chr(0xC0 | ($cp >> 6)) . chr(0x80 | ($cp & 0x3F));
	2843	+ }
	2844	+ if ($cp < 0x10000) {
	2845	+ return chr(0xE0 | ($cp >> 12)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
	2846	+ }
	2847	+ return chr(0xF0 | ($cp >> 18)) . chr(0x80 | (($cp >> 12) & 0x3F)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
1251	2848	}
1252	2849	}