← All changes | includes/class-mxchat-utils.php +1631 -85 3.2.3 → 3.2.23 View file →

		@@ -5,8 +5,378 @@
5	5
6	6	class MxChat_Utils {
7	7
8	8	/**
	9	+ * True while a storage routine is re-writing an entry it is about to re-add
	10	+ * (submit_chunked_content's clean-slate delete). delete_chunks_for_url() skips
	11	+ * the Vector Store mirror-delete while set — an internal re-store is not an
	12	+ * entry removal, and mirroring it would delete-then-reupload the entry's file
	13	+ * on every chunked save (plan 15b5c6).
	14	+ */
	15	+private static $vectorstore_mirror_suspended = false;
	16	+
	17	+/**
	18	+ * Validate a client-supplied session id (plan-mxchat-20260731-d42bec).
	19	+ *
	20	+ * sanitize_text_field() — which every session_id read site used before this —
	21	+ * preserves '/' and '..'. Harmless where the value is only an option or
	22	+ * transient key suffix, but mxchat_send_delayed_transcript() interpolates it
	23	+ * into a filesystem path, so '../../../../path/x' wrote, emailed and deleted a
	24	+ * file outside the uploads dir.
	25	+ *
	26	+ * REJECTS rather than rewrites: a silently-stripped id would orphan the
	27	+ * conversation it belongs to, which is harder to diagnose than a clean refusal.
	28	+ * Returns '' for anything malformed, so call sites fall into the empty-session
	29	+ * error paths they already have.
	30	+ *
	31	+ * The generator only ever emits 'mxchat_chat_' + 32 hex chars
	32	+ * (class-mxchat-integrator.php, js/chat-script.js), so this is not restrictive
	33	+ * in practice. Length ceiling is deliberate — session ids are also used as
	34	+ * option-name suffixes, and WP option names cap at 191 chars.
	35	+ *
	36	+ * @param mixed $raw Raw request value.
	37	+ * @return string The id if well-formed, '' otherwise.
	38	+ */
	39	+public static function sanitize_session_id($raw) {
	40	+ if (!is_scalar($raw)) {
	41	+ return '';
	42	+ }
	43	+ $val = trim((string) $raw);
	44	+ if ($val === '') {
	45	+ return '';
	46	+ }
	47	+ return preg_match('/\A[A-Za-z0-9_-]{1,128}\z/', $val) ? $val : '';
	48	+}
	49	+
	50	+/**
	51	+ * Sanitize the lead-capture consent-checkbox label (plan b062c4).
	52	+ *
	53	+ * The label is owner-supplied and renders inside the widget's email form, so
	54	+ * this is a security boundary, not a formatting nicety. One explicit
	55	+ * allowlist, used at BOTH save time (options.php sanitize + autosave AJAX)
	56	+ * and render time (widget form, admin surfaces) so the two can never drift:
	57	+ * an anchor — the whole point is "I agree to the <a>Privacy Policy</a>" —
	58	+ * plus inline emphasis. No block tags, no images, no style attributes.
	59	+ *
	60	+ * The stored consent record keeps this exact sanitized string as "the text
	61	+ * the visitor saw", so it must be deterministic: same input, same output,
	62	+ * whichever path ran it.
	63	+ *
	64	+ * @param mixed $raw Owner-entered label.
	65	+ * @return string Sanitized label, capped at 1000 chars.
	66	+ */
	67	+public static function sanitize_consent_label($raw) {
	68	+ if (!is_scalar($raw)) {
	69	+ return '';
	70	+ }
	71	+
	72	+ $allowed = array(
	73	+ 'a' => array(
	74	+ 'href' => true,
	75	+ 'title' => true,
	76	+ 'target' => true,
	77	+ 'rel' => true,
	78	+ ),
	79	+ 'strong' => array(),
	80	+ 'em' => array(),
	81	+ 'br' => array(),
	82	+ );
	83	+
	84	+ $label = wp_kses(trim((string) $raw), $allowed);
	85	+
	86	+ return mb_substr($label, 0, 1000);
	87	+}
	88	+
	89	+/**
	90	+ * Per-request cache for get_session_history(). Mirrors get_option()'s
	91	+ * request-scoped caching, which the mxchat_history_ option reads got for
	92	+ * free before plan 839c4c moved history reads onto the transcripts table.
	93	+ */
	94	+private static $history_cache = array();
	95	+
	96	+/**
	97	+ * Session chat history read from the transcripts table, in the exact array
	98	+ * shape the legacy mxchat_history_<sid> option stored (plan 839c4c). The
	99	+ * option was a second copy of state the table already held — measured
	100	+ * byte-identical in role/content/order on 174 of 177 real sessions, with
	101	+ * the table a superset on the rest — at up to 64 KB per option row. The
	102	+ * table is now the single store; nothing writes the option any more.
	103	+ *
	104	+ * Shape notes, load-bearing for the consumers:
	105	+ * - id: the transcripts row id (int). Integer ids make the pollers'
	106	+ * ">" comparisons correct where the old uniqid() strings only worked by
	107	+ * accident of hex ordering.
	108	+ * - timestamp: milliseconds, derived from the table's second-resolution GMT
	109	+ * column (x1000). Consumers comparing against a real-millisecond client
	110	+ * cutoff MUST floor the cutoff to the second and err inclusive — see the
	111	+ * persistence-off filters in class-mxchat-integrator.php.
	112	+ * - agent_name: the row's user_identifier, which the writer sets to the
	113	+ * same displayed_name value the option carried (agent name when present,
	114	+ * else email, else identifier).
	115	+ *
	116	+ * Public and static so mxchat-woo / mxchat-forms can call the same accessor
	117	+ * as core, guarded with method_exists against an older mxchat-basic.
	118	+ *
	119	+ * @param string $session_id
	120	+ * @return array[] Chronological entries: id, role, content, timestamp, agent_name.
	121	+ */
	122	+public static function get_session_history($session_id) {
	123	+ global $wpdb;
	124	+
	125	+ $session_id = self::sanitize_session_id($session_id);
	126	+ if ($session_id === '') {
	127	+ return array();
	128	+ }
	129	+
	130	+ if (array_key_exists($session_id, self::$history_cache)) {
	131	+ return self::$history_cache[$session_id];
	132	+ }
	133	+
	134	+ $table = $wpdb->prefix . 'mxchat_chat_transcripts';
	135	+
	136	+ // No SHOW TABLES guard: this is the chat hot path and the table is
	137	+ // created on activation (with an admin-load safety net). A genuinely
	138	+ // missing table fails the query and yields the same empty history the
	139	+ // old option read produced on a fresh session.
	140	+ $rows = $wpdb->get_results(
	141	+ $wpdb->prepare(
	142	+ "SELECT id, role, message, user_identifier, timestamp
	143	+ FROM `$table` WHERE session_id = %s ORDER BY id ASC",
	144	+ $session_id
	145	+ ),
	146	+ ARRAY_A
	147	+ );
	148	+
	149	+ $history = array();
	150	+ if (is_array($rows)) {
	151	+ foreach ($rows as $row) {
	152	+ // The column stores GMT (current_time('mysql', 1) at the writer),
	153	+ // so pin the parse to UTC rather than the site timezone.
	154	+ $ts = strtotime($row['timestamp'] . ' +0000');
	155	+ $history[] = array(
	156	+ 'id' => (int) $row['id'],
	157	+ 'role' => (string) $row['role'],
	158	+ 'content' => (string) $row['message'],
	159	+ 'timestamp' => ($ts ? $ts : 0) * 1000,
	160	+ 'agent_name' => (string) $row['user_identifier'],
	161	+ );
	162	+ }
	163	+ }
	164	+
	165	+ self::$history_cache[$session_id] = $history;
	166	+
	167	+ return $history;
	168	+}
	169	+
	170	+/**
	171	+ * Drop the cached history for one session (or all). The writer calls this
	172	+ * after every insert so a later read in the same request — e.g. the AI
	173	+ * context build that follows saving the user's message — sees the new row,
	174	+ * matching the read-your-own-write behavior update_option() gave the old
	175	+ * option copy.
	176	+ *
	177	+ * @param string|null $session_id Null flushes everything (test seam).
	178	+ */
	179	+public static function flush_session_history_cache($session_id = null) {
	180	+ if ($session_id === null) {
	181	+ self::$history_cache = array();
	182	+ return;
	183	+ }
	184	+
	185	+ unset(self::$history_cache[(string) $session_id]);
	186	+}
	187	+
	188	+/**
	189	+ * Append one message to a session's transcript on behalf of an add-on
	190	+ * (plan 4a664e). The Forms add-on's wizard steps travel over the add-on's
	191	+ * own AJAX action, never through the chat request, so nothing wrote them
	192	+ * to the transcript: Transcripts, the live-agent handoff card, Download
	193	+ * Transcript and every transcript email showed "[WIZARD_COMPLETE]" and no
	194	+ * answers. This is the write half of get_session_history() — same row
	195	+ * shape as the chat writer (visitor identity, captured name/email and
	196	+ * originating page from the session store, GMT timestamp), without the
	197	+ * new-session side effects (ownership, notification email, delayed
	198	+ * transcript) that only the visitor's first message should trigger.
	199	+ *
	200	+ * Public and static so add-ons can call it, guarded with method_exists
	201	+ * against an older mxchat-basic.
	202	+ *
	203	+ * @param string $session_id Chat session id.
	204	+ * @param string $role user|bot|system|agent.
	205	+ * @param string $message Stored as-is: bot rows may carry HTML, user rows
	206	+ * are rendered as text by every reader.
	207	+ * @return int The new transcripts row id, 0 when nothing was written.
	208	+ */
	209	+public static function save_session_message($session_id, $role, $message) {
	210	+ global $wpdb;
	211	+
	212	+ $session_id = self::sanitize_session_id($session_id);
	213	+ $role = is_string($role) ? strtolower(trim($role)) : '';
	214	+ $message = is_scalar($message) ? trim((string) $message) : '';
	215	+
	216	+ if ($session_id === '' || $message === ''
	217	+ || !in_array($role, array('user', 'bot', 'system', 'agent'), true)) {
	218	+ return 0;
	219	+ }
	220	+
	221	+ $table = $wpdb->prefix . 'mxchat_chat_transcripts';
	222	+
	223	+ $saved_email = MxChat_Session_Store::get($session_id, 'email');
	224	+ $saved_name = MxChat_Session_Store::get($session_id, 'name');
	225	+
	226	+ $row = array(
	227	+ 'user_id' => is_user_logged_in() ? get_current_user_id() : 0,
	228	+ 'user_identifier' => MxChat_User::mxchat_get_user_identifier(),
	229	+ 'user_email' => $saved_email ?: MxChat_User::mxchat_get_user_email(),
	230	+ 'user_name' => $saved_name ?: '',
	231	+ 'session_id' => $session_id,
	232	+ 'role' => $role,
	233	+ 'message' => $message,
	234	+ 'timestamp' => current_time('mysql', 1),
	235	+ );
	236	+
	237	+ // Same page attribution as the rest of the session; the column pair was
	238	+ // added by an upgrade routine, so tolerate its absence like the chat
	239	+ // writer does.
	240	+ $originating = MxChat_Session_Store::get($session_id, 'originating_page');
	241	+ if (is_array($originating) && !empty($originating['url'])
	242	+ && $wpdb->get_var("SHOW COLUMNS FROM `$table` LIKE 'originating_page_url'")) {
	243	+ $row['originating_page_url'] = $originating['url'];
	244	+ $row['originating_page_title'] = isset($originating['title']) ? $originating['title'] : '';
	245	+ }
	246	+
	247	+ $inserted = $wpdb->insert($table, $row);
	248	+ if ($inserted === false) {
	249	+ return 0;
	250	+ }
	251	+
	252	+ $row_id = (int) $wpdb->insert_id;
	253	+ self::flush_session_history_cache($session_id);
	254	+
	255	+ return $row_id;
	256	+}
	257	+
	258	+/**
	259	+ * Most recipients the Notification Email field will accept (plan 2f131a).
	260	+ * A settings field is not a mailing list.
	261	+ */
	262	+const NOTIFICATION_EMAIL_MAX = 5;
	263	+
	264	+/**
	265	+ * Parse the Notification Email field into a list of recipients (plan 2f131a).
	266	+ *
	267	+ * THE TRAP THIS EXISTS TO CLOSE: sanitize_email() cannot be the validator for
	268	+ * this field, because its output for the failing input is VALID. WordPress
	269	+ * strips the separator and the surplus '@' and concatenates the remains:
	270	+ *
	271	+ * [email protected], [email protected] -> [email protected]
	272	+ *
	273	+ * and is_email() then returns true on that. So every guard in the plugin passed,
	274	+ * the address was stored, the autosave ticked green, and both the new-session
	275	+ * notification and the auto-emailed transcript went to a domain that does not
	276	+ * exist — with no error anywhere. Validating the RAW part BEFORE sanitizing is
	277	+ * the whole point; reversing those two lines silently restores the bug.
	278	+ *
	279	+ * All-or-nothing by design: if any entry is bad the caller must store NOTHING.
	280	+ * A partial accept — keeping the good addresses and dropping the bad one — is
	281	+ * the same defect in a new costume, because the owner still believes everyone
	282	+ * on their list is being notified.
	283	+ *
	284	+ * @param mixed $raw Raw field value, exactly as submitted.
	285	+ * @return array{emails: string[], error: string} Empty emails + empty error
	286	+ * means the field was empty.
	287	+ */
	288	+public static function parse_notification_emails($raw) {
	289	+ $out = array('emails' => array(), 'error' => '');
	290	+
	291	+ if (!is_scalar($raw)) {
	292	+ $out['error'] = __('The notification email could not be read.', 'mxchat');
	293	+ return $out;
	294	+ }
	295	+
	296	+ $raw = trim((string) $raw);
	297	+ if ($raw === '') {
	298	+ return $out; // genuinely empty — the caller falls back to admin_email
	299	+ }
	300	+
	301	+ $seen = array();
	302	+ foreach (preg_split('/[,;]/', $raw) as $part) {
	303	+ $part = trim($part);
	304	+ if ($part === '') {
	305	+ // A trailing or doubled separator carries no address, so skipping it
	306	+ // cannot silently drop a recipient. This is the ONLY thing tolerated.
	307	+ continue;
	308	+ }
	309	+
	310	+ // RAW first. See the note above — order is load-bearing.
	311	+ $clean = is_email($part) ? sanitize_email($part) : '';
	312	+ if ($clean === '' || !is_email($clean)) {
	313	+ return array(
	314	+ 'emails' => array(),
	315	+ 'error' => sprintf(
	316	+ /* translators: %s: the email address the owner typed. */
	317	+ __('"%s" is not a valid email address, so nothing was saved. Separate multiple addresses with a comma.', 'mxchat'),
	318	+ esc_html($part)
	319	+ ),
	320	+ );
	321	+ }
	322	+
	323	+ $key = strtolower($clean);
	324	+ if (isset($seen[$key])) {
	325	+ continue; // same address twice would simply mail them twice
	326	+ }
	327	+ $seen[$key] = true;
	328	+ $out['emails'][] = $clean;
	329	+ }
	330	+
	331	+ if (count($out['emails']) > self::NOTIFICATION_EMAIL_MAX) {
	332	+ return array(
	333	+ 'emails' => array(),
	334	+ 'error' => sprintf(
	335	+ /* translators: %d: maximum number of notification recipients. */
	336	+ __('Enter at most %d email addresses, separated by commas.', 'mxchat'),
	337	+ self::NOTIFICATION_EMAIL_MAX
	338	+ ),
	339	+ );
	340	+ }
	341	+
	342	+ return $out;
	343	+}
	344	+
	345	+/**
	346	+ * The stored recipient list, ready to hand to wp_mail() (plan 2f131a).
	347	+ *
	348	+ * Fallback rule, and it is narrow on purpose: an EMPTY field falls back to the
	349	+ * site admin address, because that is the documented behaviour and an owner who
	350	+ * never filled the field in still wants their notifications. A field holding
	351	+ * something unusable does NOT fall back — it sends nowhere, exactly as before
	352	+ * this plan. Falling back on bad input would mean a typo silently redirects a
	353	+ * store's transcripts to a different mailbox than the one on screen.
	354	+ *
	355	+ * @param array|null $options mxchat_transcripts_options, or null to read it.
	356	+ * @return string[] Recipients; empty means do not send.
	357	+ */
	358	+public static function notification_recipients($options = null) {
	359	+ if (!is_array($options)) {
	360	+ $options = get_option('mxchat_transcripts_options', array());
	361	+ if (!is_array($options)) {
	362	+ $options = array();
	363	+ }
	364	+ }
	365	+
	366	+ $raw = isset($options['mxchat_notification_email']) ? $options['mxchat_notification_email'] : '';
	367	+ $raw = is_scalar($raw) ? trim((string) $raw) : '';
	368	+
	369	+ if ($raw === '') {
	370	+ $admin = get_option('admin_email');
	371	+ return is_email($admin) ? array($admin) : array();
	372	+ }
	373	+
	374	+ $parsed = self::parse_notification_emails($raw);
	375	+ return $parsed['error'] === '' ? $parsed['emails'] : array();
	376	+}
	377	+
	378	+/**
9	379	* Centralized embedding model registry. Single source of truth for dimensions
10	380	* and provider, so model-switch protection logic doesn't drift across files.
11	381	*/
12	382	public static function embedding_model_registry() {
		@@ -24,8 +394,12 @@
24	394	return isset($registry[$model]) ? (int) $registry[$model]['dims'] : 0;
25	395	}
26	396
27	397	public static function embedding_model_label($model) {
	398	+ if (is_string($model) && strpos($model, 'custom:') === 0) {
	399	+ /* translators: %s: the embedding model name configured on the custom provider */
	400	+ return sprintf(__('%s (custom provider)', 'mxchat'), substr($model, 7));
	401	+ }
28	402	$registry = self::embedding_model_registry();
29	403	return isset($registry[$model]) ? $registry[$model]['label'] : $model;
30	404	}
31	405
		@@ -48,8 +422,110 @@
48	422	}
49	423	}
50	424
51	425	/**
	426	+ * The model name the custom-provider embedding path will send, mirroring the
	427	+ * fallback chain the request itself uses: dedicated custom embedding model,
	428	+ * else the custom chat model, else 'default'. Single source shared by
	429	+ * generate_embedding_custom() and the mismatch-warning "selected" side so the
	430	+ * two can never drift (plan ae02cb).
	431	+ */
	432	+public static function resolve_custom_embedding_model($options) {
	433	+ if (isset($options['custom_provider_embedding_model']) && trim((string) $options['custom_provider_embedding_model']) !== '') {
	434	+ return trim((string) $options['custom_provider_embedding_model']);
	435	+ }
	436	+ if (isset($options['custom_provider_model']) && trim((string) $options['custom_provider_model']) !== '') {
	437	+ return trim((string) $options['custom_provider_model']);
	438	+ }
	439	+ return 'default';
	440	+}
	441	+
	442	+/**
	443	+ * The EFFECTIVE selected embedding model — what the next embed will actually
	444	+ * use. With custom-provider embeddings on this is the custom identity in the
	445	+ * same 'custom:<model>' form stamp_active_embedding_model() records, not the
	446	+ * inert standard dropdown value. Mismatch-warning comparisons must read this,
	447	+ * never $options['embedding_model'] directly — the dropdown cannot be
	448	+ * deselected, so reading it raw flags every correctly-configured custom setup.
	449	+ */
	450	+public static function get_selected_embedding_model($options = null) {
	451	+ if (!is_array($options)) {
	452	+ $options = get_option('mxchat_options', array());
	453	+ }
	454	+ if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
	455	+ return 'custom:' . self::resolve_custom_embedding_model($options);
	456	+ }
	457	+ return $options['embedding_model'] ?? '';
	458	+}
	459	+
	460	+/**
	461	+ * Extract the 11-character YouTube video ID from a URL, or '' if the URL is
	462	+ * not a single-video YouTube link. Single source of truth for both the KB
	463	+ * ingestion side and the chat render side — do not duplicate this parsing.
	464	+ * Channel, playlist, and search URLs deliberately return '' (only a URL that
	465	+ * identifies one video can be embedded).
	466	+ */
	467	+public static function parse_youtube_id($url) {
	468	+ if (!is_string($url) || $url === '' || !preg_match('#^https?://#i', $url)) {
	469	+ return '';
	470	+ }
	471	+ $host = strtolower((string) wp_parse_url($url, PHP_URL_HOST));
	472	+ $host = preg_replace('/^(www|m)\./', '', $host);
	473	+ $path = (string) wp_parse_url($url, PHP_URL_PATH);
	474	+ $id = '';
	475	+ if ($host === 'youtu.be') {
	476	+ $segments = explode('/', ltrim($path, '/'));
	477	+ $id = $segments[0] ?? '';
	478	+ } elseif (in_array($host, array('youtube.com', 'youtube-nocookie.com'), true)) {
	479	+ if (preg_match('#^/(?:shorts|embed|live|v)/([A-Za-z0-9_-]+)#', $path, $m)) {
	480	+ $id = $m[1];
	481	+ } elseif ($path === '/watch') {
	482	+ parse_str((string) wp_parse_url($url, PHP_URL_QUERY), $query_vars);
	483	+ $id = isset($query_vars['v']) ? (string) $query_vars['v'] : '';
	484	+ }
	485	+ }
	486	+ $id = preg_replace('/[^A-Za-z0-9_-]/', '', (string) $id);
	487	+ return (strlen($id) === 11) ? $id : '';
	488	+}
	489	+
	490	+/**
	491	+ * plan-mxchat-20260813-f52492 — video-card gating.
	492	+ *
	493	+ * Two standalone options (NOT mxchat_options — they skip the sanitize/autosave
	494	+ * traps entirely), read here so the gate in the integrator and the fields on
	495	+ * Knowledge -> Chunking & Retrieval can never disagree about a default.
	496	+ *
	497	+ * Master switch. Default ON: the card is existing behavior, and this is an
	498	+ * opt-out for owners who never want one, not a new feature to opt into.
	499	+ */
	500	+public static function video_embed_enabled() {
	501	+ return get_option('mxchat_video_embed_enabled', 'on') === 'on';
	502	+}
	503	+
	504	+/**
	505	+ * The video card's OWN confidence floor, as a 0-1 cosine — deliberately not
	506	+ * the site-wide Similarity Threshold (default 35). "Good enough to quote in
	507	+ * the answer" and "good enough to put a video on screen" are different
	508	+ * questions: retrieval is allowed to be generous because the model still
	509	+ * decides what to say, whereas the card is asserted to the visitor with no
	510	+ * such filter. Stored as an int percentage to match the site-wide slider's
	511	+ * convention; the default (55) sits above it on purpose.
	512	+ *
	513	+ * MXCHAT_VIDEO_EMBED_THRESHOLD_DEFAULT is the single source of that number.
	514	+ */
	515	+public static function video_embed_threshold() {
	516	+ $stored = get_option('mxchat_video_embed_threshold', null);
	517	+ $percent = ($stored === null || $stored === '')
	518	+ ? MXCHAT_VIDEO_EMBED_THRESHOLD_DEFAULT
	519	+ : (int) $stored;
	520	+ if ($percent < 0) { $percent = 0; }
	521	+ if ($percent > 100) { $percent = 100; }
	522	+ // Cast: PHP evaluates 100/100 to int(1), so an unclamped return type would
	523	+ // vary with the stored value. Callers compare against a cosine — keep it float.
	524	+ return (float) $percent / 100;
	525	+}
	526	+
	527	+/**
52	528	* UPDATED: Submit or update content (and its embedding) in the database.
53	529	* Stores in Pinecone if enabled, otherwise stores in WordPress DB.
54	530	*
55	531	* @param string $content The content to be embedded.
		@@ -59,9 +535,9 @@
59	535	* @param string $bot_id The bot ID for multi-bot support
60	536	* @param string $content_type The type of content (post, page, pdf, url, manual, product, etc.)
61	537	* @return bool|WP_Error True on success, WP_Error on failure
62	538	*/
63		-public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
	539	+public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content', $title = '') {
64	540	global $wpdb;
65	541	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
66	542
67	543	//error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ', Type: ' . $content_type . ')');
		@@ -66,10 +542,18 @@
66	542
67	543	//error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ', Type: ' . $content_type . ')');
68	544	//error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
69	545
70		- // Sanitize the source URL
71		- $source_url = esc_url_raw($source_url);
	546	+ // Sanitize the source URL. Internal identities (mxchat:// manual docs,
	547	+ // upload:// file uploads) are NOT web URLs — esc_url_raw EMPTIES them
	548	+ // because its protocol list is statically cached and effectively
	549	+ // unfilterable (measured, plan 945406 / 0485e5) — sanitize those as text
	550	+ // so upserts stay keyed to a stable identity across re-imports.
	551	+ if (preg_match('#^(mxchat|upload)://#i', $source_url)) {
	552	+ $source_url = sanitize_text_field($source_url);
	553	+ } else {
	554	+ $source_url = esc_url_raw($source_url);
	555	+ }
72	556
73	557	// Sanitize content_type
74	558	$content_type = sanitize_key($content_type);
75	559	if (empty($content_type)) {
		@@ -75,8 +559,14 @@
75	559	if (empty($content_type)) {
76	560	$content_type = 'content'; // Fallback for backwards compatibility
77	561	}
78	562
	563	+ // Page title captured by the importer (plan d9ee66), registered under the
	564	+ // SANITIZED url so the storage writers below find it by the same key.
	565	+ if (is_string($title) && trim($title) !== '') {
	566	+ self::set_source_title($source_url, $title);
	567	+ }
	568	+
79	569	// Just ensure UTF-8 validity without aggressive escaping
80	570	$safe_content = wp_check_invalid_utf8($content);
81	571	// Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D)
82	572	$safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
		@@ -84,9 +574,16 @@
84	574	// Check if chunking should be applied
85	575	$chunker = MxChat_Chunker::from_settings();
86	576	if ($chunker->should_chunk($safe_content)) {
87	577	//error_log('[MXCHAT-DB] Content exceeds chunk threshold, using chunked submission');
88		- return self::submit_chunked_content($safe_content, $source_url, $api_key, $bot_id, $content_type, $chunker);
	578	+ $chunk_result = self::submit_chunked_content($safe_content, $source_url, $api_key, $bot_id, $content_type, $chunker);
	579	+ // Vector Store mirror gets the entry WHOLE — one file per KB entry,
	580	+ // OpenAI chunks server-side; local chunking is our own embedding
	581	+ // concern and never reaches the store (plan 15b5c6).
	582	+ if ($chunk_result === true && class_exists('MxChat_Vectorstore_Manager')) {
	583	+ MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type);
	584	+ }
	585	+ return $chunk_result;
89	586	}
90	587
91	588	// UPDATED: Generate the embedding using bot-specific configuration
92	589	$embedding_vector = self::generate_embedding($content, $api_key, $bot_id);
		@@ -91,10 +588,13 @@
91	588	// UPDATED: Generate the embedding using bot-specific configuration
92	589	$embedding_vector = self::generate_embedding($content, $api_key, $bot_id);
93	590
94	591	if (!is_array($embedding_vector)) {
95		- //error_log('[MXCHAT-DB] Error: Embedding generation failed');
96		- return new WP_Error('embedding_failed', 'Failed to generate embedding for content');
	592	+ // Surface the provider's real reason instead of a fixed string (4a7c0a).
	593	+ $reason = is_wp_error($embedding_vector)
	594	+ ? $embedding_vector->get_error_message()
	595	+ : 'Failed to generate embedding for content';
	596	+ return new WP_Error('embedding_failed', $reason);
97	597	}
98	598
99	599	//error_log('[MXCHAT-DB] Embedding generated successfully');
100	600
		@@ -101,18 +601,175 @@
101	601	// UPDATED: Check if Pinecone is enabled for this specific bot
102	602	if (self::is_pinecone_enabled_for_bot($bot_id)) {
103	603	//error_log('[MXCHAT-DB] Pinecone is enabled for bot ' . $bot_id . ' - using Pinecone storage');
104	604	// Store in Pinecone only
105		- return self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id, $content_type);
	605	+ $result = self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id, $content_type);
	606	+ // If this URL previously stored as CHUNKED content and the new content fits in a
	607	+ // single vector, the upsert above only overwrote the base id — the old
	608	+ // md5(url)_chunk_N vectors would keep serving the stale text. Sweep them.
	609	+ // Only for a real source_url: chunk ids derive from it, and md5('') is shared
	610	+ // by legacy URL-less entries so a blind sweep there could hit other entries.
	611	+ if ($result === true && !empty($source_url)) {
	612	+ self::cleanup_pinecone_chunk_stragglers($source_url, $bot_id);
	613	+ }
	614	+ if ($result === true && class_exists('MxChat_Vectorstore_Manager')) {
	615	+ MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type);
	616	+ }
	617	+ return $result;
106	618	} else {
107	619	//error_log('[MXCHAT-DB] Pinecone not enabled for bot ' . $bot_id . ' - using WordPress storage');
108	620	// Store in WordPress database only
109	621	$embedding_vector_serialized = maybe_serialize($embedding_vector);
110		- return self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type);
	622	+ $result = self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type);
	623	+ if ($result === true && class_exists('MxChat_Vectorstore_Manager')) {
	624	+ MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type);
	625	+ }
	626	+ return $result;
111	627	}
112	628	}
113	629
	630	+/* -------------------------------------------------------------------------
	631	+ * Source titles (3.2.23, plan d9ee66)
	632	+ *
	633	+ * A knowledge entry imported from a URL or sitemap remembers the page's own
	634	+ * title, so an add-on that names a source (AI Search's source cards) can show
	635	+ * "About Us" instead of a guess from the slug ("Aboutus"). Stored in the
	636	+ * `title` column of the WP knowledge table and as `title` metadata on Pinecone
	637	+ * vectors / documents. The importer hands the title to submit_content_to_db(),
	638	+ * which registers it under the source URL; the writers look it up by that key.
	639	+ * ---------------------------------------------------------------------- */
	640	+
	641	+/** @var array<string,string> source_url => title, for the current request */
	642	+private static $source_titles = array();
	643	+
	644	+public static function set_source_title($source_url, $title) {
	645	+ $key = trim((string) $source_url);
	646	+ $title = self::clean_title($title);
	647	+ if ($key === '' || $title === '') {
	648	+ return;
	649	+ }
	650	+ self::$source_titles[$key] = $title;
	651	+}
	652	+
	653	+public static function source_title_for($source_url) {
	654	+ $key = trim((string) $source_url);
	655	+ return ($key !== '' && isset(self::$source_titles[$key])) ? self::$source_titles[$key] : '';
	656	+}
	657	+
	658	+/** Entities decoded, tags gone, whitespace collapsed, capped at the column width. */
	659	+public static function clean_title($title) {
	660	+ $title = html_entity_decode((string) $title, ENT_QUOTES | ENT_HTML5, 'UTF-8');
	661	+ $title = wp_strip_all_tags($title);
	662	+ $title = trim((string) preg_replace('/\s+/u', ' ', $title));
	663	+ if (mb_strlen($title, 'UTF-8') > 255) {
	664	+ $title = rtrim(mb_substr($title, 0, 255, 'UTF-8'));
	665	+ }
	666	+ return $title;
	667	+}
	668	+
114	669	/**
	670	+ * The page's own title from fetched HTML: og:title, else <title> with a
	671	+ * trailing "| Site Name" style suffix removed, else the first <h1>. Returns ''
	672	+ * when nothing usable is there, so the caller keeps its own fallback.
	673	+ */
	674	+public static function extract_page_title($html) {
	675	+ $html = (string) $html;
	676	+ if ($html === '') {
	677	+ return '';
	678	+ }
	679	+ $head = substr($html, 0, 200000);
	680	+ $site = '';
	681	+ if (preg_match('/<meta\s+[^>]*property=["\']og:site_name["\'][^>]*content=["\']([^"\']{1,120})["\']/i', $head, $m)
	682	+ || preg_match('/<meta\s+[^>]*content=["\']([^"\']{1,120})["\'][^>]*property=["\']og:site_name["\']/i', $head, $m)) {
	683	+ $site = self::clean_title($m[1]);
	684	+ }
	685	+ $candidate = '';
	686	+ if (preg_match('/<meta\s+[^>]*property=["\']og:title["\'][^>]*content=["\']([^"\']{1,500})["\']/i', $head, $m)
	687	+ || preg_match('/<meta\s+[^>]*content=["\']([^"\']{1,500})["\'][^>]*property=["\']og:title["\']/i', $head, $m)) {
	688	+ $candidate = self::clean_title($m[1]);
	689	+ }
	690	+ if ($candidate === '' && preg_match('/<title[^>]*>(.*?)<\/title>/is', $head, $m)) {
	691	+ $candidate = self::clean_title($m[1]);
	692	+ }
	693	+ if ($candidate !== '') {
	694	+ $candidate = self::strip_site_suffix($candidate, $site);
	695	+ }
	696	+ if ($candidate === '' && preg_match('/<h1[^>]*>(.*?)<\/h1>/is', $html, $m)) {
	697	+ $candidate = self::clean_title($m[1]);
	698	+ }
	699	+ return $candidate;
	700	+}
	701	+
	702	+/**
	703	+ * "About Us | Just Facts" -> "About Us". Only a TRAILING segment after a spaced
	704	+ * separator goes, only when what is left is a real title (3+ characters) and the
	705	+ * removed part is short enough to be a site name; a known og:site_name wins.
	706	+ * Byte offsets throughout: every separator starts with an ASCII space, so a cut
	707	+ * there never lands inside a multibyte character.
	708	+ */
	709	+private static function strip_site_suffix($title, $site_name = '') {
	710	+ $seps = array(' | ', ' – ', ' — ', ' - ', ' :: ', ' · ', ' » ', ' • ');
	711	+ if ($site_name !== '') {
	712	+ foreach ($seps as $sep) {
	713	+ $tail = $sep . $site_name;
	714	+ if (strlen($title) > strlen($tail) && substr($title, -strlen($tail)) === $tail) {
	715	+ return trim(substr($title, 0, strlen($title) - strlen($tail)));
	716	+ }
	717	+ }
	718	+ }
	719	+ $best_pos = -1;
	720	+ $best_sep = '';
	721	+ foreach ($seps as $sep) {
	722	+ $pos = strrpos($title, $sep);
	723	+ if ($pos !== false && $pos > $best_pos) {
	724	+ $best_pos = $pos;
	725	+ $best_sep = $sep;
	726	+ }
	727	+ }
	728	+ if ($best_pos > 0) {
	729	+ $left = trim(substr($title, 0, $best_pos));
	730	+ $right = trim(substr($title, $best_pos + strlen($best_sep)));
	731	+ if (mb_strlen($left, 'UTF-8') >= 3 && $right !== '' && mb_strlen($right, 'UTF-8') <= 60) {
	732	+ return $left;
	733	+ }
	734	+ }
	735	+ return $title;
	736	+}
	737	+
	738	+/**
	739	+ * Make sure the knowledge table has the `title` column (3.2.23, plan d9ee66).
	740	+ * Activation adds it; this is the safety net for a copy updated in place, and
	741	+ * the one place add-ons ask "may I read/write title?". A positive answer is
	742	+ * remembered in an option, so after the first time this costs one option read.
	743	+ */
	744	+public static function ensure_kb_title_column() {
	745	+ global $wpdb;
	746	+ static $ready = null;
	747	+ if ($ready !== null) {
	748	+ return $ready;
	749	+ }
	750	+ if (get_option('mxchat_kb_title_column') === '1') {
	751	+ $ready = true;
	752	+ return true;
	753	+ }
	754	+ $table = $wpdb->prefix . 'mxchat_system_prompt_content';
	755	+ $cols = $wpdb->get_col("SHOW COLUMNS FROM {$table}", 0);
	756	+ if (empty($cols)) {
	757	+ $ready = false;
	758	+ return false;
	759	+ }
	760	+ if (!in_array('title', $cols, true)) {
	761	+ $wpdb->query("ALTER TABLE {$table} ADD COLUMN title VARCHAR(255) DEFAULT NULL AFTER source_url");
	762	+ $cols = $wpdb->get_col("SHOW COLUMNS FROM {$table}", 0);
	763	+ }
	764	+ $ready = in_array('title', $cols, true);
	765	+ if ($ready) {
	766	+ update_option('mxchat_kb_title_column', '1', false);
	767	+ }
	768	+ return $ready;
	769	+}
	770	+
	771	+/**
115	772	* UPDATED: Check if Pinecone is enabled and properly configured for a specific bot
116	773	*/
117	774	private static function is_pinecone_enabled_for_bot($bot_id = 'default') {
118	775	// For default bot or when multi-bot is not active, use original method
		@@ -224,11 +881,15 @@
224	881	// Check if this is truly manual content (no URL at all) vs a real URL that filter_var rejects
225	882	// filter_var(FILTER_VALIDATE_URL) rejects valid URLs with encoded chars, non-ASCII, fragments, etc.
226	883	// Use a looser check: if it starts with http(s):// or has a scheme, it's a URL
227	884	$has_url_scheme = !empty($source_url) && preg_match('#^https?://#i', $source_url);
	885	+ // upload:// identities (admin document/PDF uploads, plan 0485e5) are stable
	886	+ // and deduplicable — treat them like URLs so a re-upload UPDATES the row
	887	+ // instead of minting a fresh manual identity (which would duplicate).
	888	+ $has_stable_identity = $has_url_scheme || (!empty($source_url) && preg_match('#^upload://#i', $source_url));
228	889	// Treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
229	890	$is_legacy_mxchat_url = $has_url_scheme && strpos($source_url, 'mxchat.ai') !== false;
230		- $is_manual_content = empty($source_url) || $source_url === '' || !$has_url_scheme || $is_legacy_mxchat_url;
	891	+ $is_manual_content = empty($source_url) || $source_url === '' || !$has_stable_identity || $is_legacy_mxchat_url;
231	892
232	893	if ($is_manual_content) {
233	894	// Generate unique identifier for manual content to prevent overwrites
234	895	$source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
		@@ -249,8 +910,13 @@
249	910	//error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)');
250	911	}
251	912	// ===== END FIX =====
252	913
	914	+ // Page title captured at import (plan d9ee66): written only when the
	915	+ // importer supplied one, so a re-index without a title keeps the old value.
	916	+ $title_value = $is_manual_content ? '' : self::source_title_for($source_url);
	917	+ $title_ready = ($title_value !== '') && self::ensure_kb_title_column();
	918	+
253	919	// Progressive fallback mechanism for problematic content
254	920	$attempt = 1;
255	921	$max_attempts = 3;
256	922	$current_content = $safe_content;
		@@ -255,10 +921,8 @@
255	921	$max_attempts = 3;
256	922	$current_content = $safe_content;
257	923	$result = false;
258	924
259		- $active_model = self::get_active_embedding_model();
260		-
261	925	while ($attempt <= $max_attempts && $result === false) {
262	926	try {
263	927	if ($existing_id) {
264	928	//error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
		@@ -263,41 +927,41 @@
263	927	if ($existing_id) {
264	928	//error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
265	929
266	930	// Update the existing row - UPDATED 2.5.6: Added content_type
267		- $result = $wpdb->update(
268		- $table_name,
269		- array(
270		- 'url' => $source_url,
271		- 'article_content' => $current_content,
272		- 'embedding_vector' => $embedding_vector_serialized,
273		- 'source_url' => $source_url,
274		- 'content_type' => $content_type,
275		- 'embedding_model' => $active_model,
276		- 'timestamp' => current_time('mysql'),
277		- ),
278		- array('id' => $existing_id),
279		- array('%s','%s','%s','%s','%s','%s','%s'),
280		- array('%d')
	931	+ $row = array(
	932	+ 'url' => $source_url,
	933	+ 'article_content' => $current_content,
	934	+ 'embedding_vector' => $embedding_vector_serialized,
	935	+ 'source_url' => $source_url,
	936	+ 'content_type' => $content_type,
	937	+ 'timestamp' => current_time('mysql'),
281	938	);
	939	+ $formats = array('%s','%s','%s','%s','%s','%s');
	940	+ if ($title_ready) {
	941	+ $row['title'] = $title_value;
	942	+ $formats[] = '%s';
	943	+ }
	944	+ $result = $wpdb->update($table_name, $row, array('id' => $existing_id), $formats, array('%d'));
282	945	} else {
283	946	//error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
284	947	//error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
285	948
286	949	// Insert a new row - UPDATED 2.5.6: Added content_type
287		- $result = $wpdb->insert(
288		- $table_name,
289		- array(
290		- 'url' => $source_url, // Now unique for manual content
291		- 'article_content' => $current_content,
292		- 'embedding_vector' => $embedding_vector_serialized,
293		- 'source_url' => $source_url, // Now unique for manual content
294		- 'content_type' => $content_type,
295		- 'embedding_model' => $active_model,
296		- 'timestamp' => current_time('mysql'),
297		- ),
298		- array('%s','%s','%s','%s','%s','%s','%s')
	950	+ $row = array(
	951	+ 'url' => $source_url, // Now unique for manual content
	952	+ 'article_content' => $current_content,
	953	+ 'embedding_vector' => $embedding_vector_serialized,
	954	+ 'source_url' => $source_url, // Now unique for manual content
	955	+ 'content_type' => $content_type,
	956	+ 'timestamp' => current_time('mysql'),
299	957	);
	958	+ $formats = array('%s','%s','%s','%s','%s','%s');
	959	+ if ($title_ready) {
	960	+ $row['title'] = $title_value;
	961	+ $formats[] = '%s';
	962	+ }
	963	+ $result = $wpdb->insert($table_name, $row, $formats);
300	964	}
301	965
302	966	if ($result === false) {
303	967	//error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . ')');
		@@ -356,10 +1020,11 @@
356	1020	// ===== UPDATED: Handle manual content with unique vector IDs =====
357	1021	if ($vector_id) {
358	1022	// Use provided vector ID
359	1023	//error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
360		- } elseif (!empty($url) && preg_match('#^https?://#i', $url)) {
361		- // For URLs, use URL-based ID (existing behavior)
	1024	+ } elseif (!empty($url) && preg_match('#^(https?|upload)://#i', $url)) {
	1025	+ // For URLs — and stable upload:// identities (plan 0485e5) — use an
	1026	+ // identity-derived ID so a re-import upserts the same vector.
362	1027	$vector_id = md5($url);
363	1028	//error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
364	1029	} else {
365	1030	// For manual content (empty/no URL scheme), generate unique ID
		@@ -419,11 +1084,24 @@
419	1084	'type' => $content_type, // Now supports: post, page, pdf, url, manual, product, etc.
420	1085	'last_updated' => time(),
421	1086	'created_at' => time(), // Add creation timestamp
422	1087	'bot_id' => $bot_id, // Add bot identification
423		- 'embedding_model' => self::get_active_embedding_model() // 3.2.3: track which model produced this vector
424	1088	);
425		-
	1089	+ $page_title = self::source_title_for($url);
	1090	+ if ($page_title !== '') {
	1091	+ $metadata['title'] = $page_title; // page title captured at import (plan d9ee66)
	1092	+ }
	1093	+
	1094	+ // Document index (plan 362c31): same id, same metadata, through the
	1095	+ // documents API. A site on the classic vector index never enters here.
	1096	+ if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) {
	1097	+ return MxChat_Pinecone_Documents::upsert_one($vector_id, $content, $embedding_vector, $metadata, array(
	1098	+ 'api_key' => $api_key,
	1099	+ 'host' => $host,
	1100	+ 'namespace' => $namespace,
	1101	+ ));
	1102	+ }
	1103	+
426	1104	$vector_data = array(
427	1105	'id' => $vector_id,
428	1106	'values' => $embedding_vector,
429	1107	'metadata' => $metadata
		@@ -478,8 +1156,104 @@
478	1156	return true;
479	1157	}
480	1158
481	1159	/**
	1160	+ * Caller-side pre-flight for KB ingestion: can an embedding request be made
	1161	+ * with these options, and which API key should travel downstream?
	1162	+ *
	1163	+ * Custom-provider-aware — generate_embedding() below routes to the custom
	1164	+ * endpoint FIRST and ignores the passed cloud key entirely when
	1165	+ * custom_provider_for_embeddings is on, so on that branch the only real
	1166	+ * requirement is a Base URL. Ingestion callers that gated on a cloud API key
	1167	+ * were killing keyless custom-embeddings sites (local Ollama / LM Studio
	1168	+ * class) before the embed layer could route (plan cbd5fd).
	1169	+ *
	1170	+ * NOTE: reads $options['embedding_model'] raw on purpose — this mirrors
	1171	+ * generate_embedding()'s own routing read, NOT the mismatch-banner's
	1172	+ * "selected" chain (get_selected_embedding_model). The helper must predict
	1173	+ * what the very next embed call will do, byte-for-byte.
	1174	+ *
	1175	+ * Decision only — callers keep their own error-surfacing shape (admin-notice
	1176	+ * transient + redirect, wp_send_json_error, WP_Error, silent return).
	1177	+ *
	1178	+ * @param array|null $options Resolved options (bot-specific where the caller
	1179	+ * has them); null loads the default bot's options.
	1180	+ * @return array {
	1181	+ * @type bool $ok Whether ingestion can proceed.
	1182	+ * @type string $api_key Key to pass downstream ('' on the custom branch —
	1183	+ * generate_embedding() ignores it there).
	1184	+ * @type string $reason Human-readable blocker; '' when $ok.
	1185	+ * @type string $provider Short provider label ('OpenAI', 'Voyage AI',
	1186	+ * 'Google Gemini', 'Custom Provider').
	1187	+ * }
	1188	+ */
	1189	+public static function embedding_preflight($options = null) {
	1190	+ if (!is_array($options)) {
	1191	+ $options = get_option('mxchat_options');
	1192	+ $options = is_array($options) ? $options : array();
	1193	+ }
	1194	+
	1195	+ // Custom branch mirrors generate_embedding()'s routing order (custom first).
	1196	+ if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
	1197	+ $base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
	1198	+ if ($base_url === '') {
	1199	+ return array(
	1200	+ 'ok' => false,
	1201	+ 'api_key' => '',
	1202	+ // Same string generate_embedding_custom() returns for this state.
	1203	+ 'reason' => __('Custom provider Base URL is not configured.', 'mxchat'),
	1204	+ 'provider' => 'Custom Provider',
	1205	+ );
	1206	+ }
	1207	+ return array('ok' => true, 'api_key' => '', 'reason' => '', 'provider' => 'Custom Provider');
	1208	+ }
	1209	+
	1210	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	1211	+ if (strpos($selected_model, 'voyage') === 0) {
	1212	+ $api_key = $options['voyage_api_key'] ?? '';
	1213	+ $provider = 'Voyage AI';
	1214	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	1215	+ $api_key = $options['gemini_api_key'] ?? '';
	1216	+ $provider = 'Google Gemini';
	1217	+ } else {
	1218	+ $api_key = $options['api_key'] ?? '';
	1219	+ $provider = 'OpenAI';
	1220	+ }
	1221	+
	1222	+ if (empty($api_key)) {
	1223	+ return array(
	1224	+ 'ok' => false,
	1225	+ 'api_key' => '',
	1226	+ 'reason' => sprintf(
	1227	+ /* translators: %s: embedding provider name */
	1228	+ __('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'),
	1229	+ $provider
	1230	+ ),
	1231	+ 'provider' => $provider,
	1232	+ );
	1233	+ }
	1234	+
	1235	+ return array('ok' => true, 'api_key' => $api_key, 'reason' => '', 'provider' => $provider);
	1236	+}
	1237	+
	1238	+/**
	1239	+ * Public QUERY-side entry point (plan 876edb). The chat pipeline's
	1240	+ * MxChat_Integrator::mxchat_generate_embedding() adapter routes through here
	1241	+ * so the query and index sides share ONE provider-routing implementation —
	1242	+ * the same endpoints, request bodies, and stamping semantics. The Integrator
	1243	+ * keeps its own error vocabulary by translating the WP_Error this returns
	1244	+ * (see the structured error data on every failure path below).
	1245	+ *
	1246	+ * @param string $text The text to be embedded.
	1247	+ * @param string $api_key Caller-resolved API key (per-bot on the query side).
	1248	+ * @param string $bot_id The bot ID for multi-bot support.
	1249	+ * @return array|WP_Error The embedding vector, or WP_Error carrying the reason.
	1250	+ */
	1251	+public static function generate_query_embedding($text, $api_key, $bot_id = 'default') {
	1252	+ return self::generate_embedding($text, $api_key, $bot_id);
	1253	+}
	1254	+
	1255	+/**
482	1256	* UPDATED: Generate an embedding for the given text using bot-specific configuration.
483	1257	*
484	1258	* @param string $text The text to be embedded.
485	1259	* @param string $api_key The API key used for generating embeddings.
		@@ -493,11 +1267,24 @@
493	1267	} else {
494	1268	$bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
495	1269	$options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
496	1270	}
497		-
	1271	+
	1272	+ // Opt-in: when the custom provider is selected for embeddings, route the KB
	1273	+ // INDEX side through the same custom endpoint the query side uses, so stored
	1274	+ // vectors and query vectors come from the same model. Default-off behavior
	1275	+ // below is untouched.
	1276	+ if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
	1277	+ $custom = self::generate_embedding_custom($text, $options);
	1278	+ // The custom path already returns a human-readable error string —
	1279	+ // carry it instead of collapsing to null (plan 4a7c0a). The 'custom'
	1280	+ // branch marker lets the Integrator adapter map the string back onto
	1281	+ // its own error codes (876edb).
	1282	+ return is_array($custom) ? $custom : new WP_Error('embedding_failed', (string) $custom, array('branch' => 'custom'));
	1283	+ }
	1284	+
498	1285	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
499		-
	1286	+
500	1287	// Determine endpoint and API key based on model
501	1288	if (strpos($selected_model, 'voyage') === 0) {
502	1289	$endpoint = 'https://api.voyageai.com/v1/embeddings';
503	1290	$api_key = $options['voyage_api_key'] ?? '';
		@@ -505,10 +1292,13 @@
505	1292	$endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
506	1293	$api_key = $options['gemini_api_key'] ?? '';
507	1294	} else {
508	1295	$endpoint = 'https://api.openai.com/v1/embeddings';
509		- // Use the bot-specific API key or fallback to passed API key
510		- $api_key = $options['api_key'] ?? $api_key;
	1296	+ // Prefer the caller-resolved key when one was passed — the query side
	1297	+ // resolves per-bot keys at its call sites (integrator adapter, 876edb).
	1298	+ // Index callers pass the preflight key, which equals this options read,
	1299	+ // so nothing changes for them.
	1300	+ $api_key = !empty($api_key) ? $api_key : ($options['api_key'] ?? '');
511	1301	}
512	1302
513	1303	// Prepare request body based on provider
514	1304	if (strpos($selected_model, 'gemini-embedding') === 0) {
		@@ -559,14 +1349,22 @@
559	1349
560	1350	$response = wp_remote_post($endpoint, $args);
561	1351
562	1352	if (is_wp_error($response)) {
563		- //error_log('Error generating embedding for bot ' . $bot_id . ': ' . $response->get_error_message());
564		- return null;
	1353	+ $message = 'Embedding request failed (connection): ' . $response->get_error_message();
	1354	+ if (class_exists('MxChat_Admin')) {
	1355	+ MxChat_Admin::mxchat_log_debug('embedding_error', $message, array('model' => $selected_model, 'bot_id' => $bot_id));
	1356	+ }
	1357	+ return new WP_Error('embedding_failed', $message, array(
	1358	+ 'branch' => 'cloud',
	1359	+ 'kind' => 'connection',
	1360	+ 'reason' => $response->get_error_message(),
	1361	+ 'model' => $selected_model,
	1362	+ ));
565	1363	}
566		-
	1364	+
567	1365	$response_body = json_decode(wp_remote_retrieve_body($response), true);
568		-
	1366	+
569	1367	// Handle different response formats based on provider
570	1368	if (strpos($selected_model, 'gemini-embedding') === 0) {
571	1369	// Gemini API response format
572	1370	if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
		@@ -572,10 +1370,9 @@
572	1370	if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
573	1371	self::stamp_active_embedding_model($selected_model);
574	1372	return $response_body['embedding']['values'];
575	1373	} else {
576		- //error_log('Invalid response received from Gemini embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
577		- return null;
	1374	+ return self::embedding_failure_error($response, $selected_model, $api_key, $bot_id);
578	1375	}
579	1376	} else {
580	1377	// OpenAI/Voyage API response format
581	1378	if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
		@@ -581,15 +1378,179 @@
581	1378	if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
582	1379	self::stamp_active_embedding_model($selected_model);
583	1380	return $response_body['data'][0]['embedding'];
584	1381	} else {
585		- //error_log('Invalid response received from embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
586		- return null;
	1382	+ return self::embedding_failure_error($response, $selected_model, $api_key, $bot_id);
587	1383	}
588	1384	}
589	1385	}
590	1386
591	1387	/**
	1388	+ * Build a WP_Error carrying the embedding provider's REAL failure reason,
	1389	+ * and record it in the Debug Mode log. Previously every failure path
	1390	+ * returned bare null, so customers saw only "Failed to generate embedding
	1391	+ * for content" / "Failed to store any chunks" with no cause (plan 4a7c0a).
	1392	+ *
	1393	+ * The API key never appears in provider response bodies (it travels in the
	1394	+ * request headers), but the reason is scrubbed for it anyway before it can
	1395	+ * reach a notice or the debug log.
	1396	+ */
	1397	+private static function embedding_failure_error($response, $selected_model, $api_key, $bot_id) {
	1398	+ $status = (int) wp_remote_retrieve_response_code($response);
	1399	+ $raw = (string) wp_remote_retrieve_body($response);
	1400	+ $decoded = json_decode($raw, true);
	1401	+
	1402	+ // Provider error shapes: OpenAI + Gemini use {"error":{"message":…}};
	1403	+ // Voyage uses {"detail":…}.
	1404	+ $reason = '';
	1405	+ if (is_array($decoded)) {
	1406	+ if (isset($decoded['error']['message']) && is_string($decoded['error']['message'])) {
	1407	+ $reason = $decoded['error']['message'];
	1408	+ } elseif (isset($decoded['detail']) && is_string($decoded['detail'])) {
	1409	+ $reason = $decoded['detail'];
	1410	+ }
	1411	+ }
	1412	+ if ($reason === '') {
	1413	+ $reason = ($raw !== '') ? substr($raw, 0, 200) : 'empty or malformed response';
	1414	+ }
	1415	+ if (is_string($api_key) && $api_key !== '') {
	1416	+ $reason = str_replace($api_key, '[redacted]', $reason);
	1417	+ }
	1418	+ $reason = substr($reason, 0, 300);
	1419	+ $message = sprintf('Embedding failed (%s, HTTP %d): %s', $selected_model, $status, $reason);
	1420	+
	1421	+ if (class_exists('MxChat_Admin')) {
	1422	+ MxChat_Admin::mxchat_log_debug('embedding_error', $message, array(
	1423	+ 'model' => $selected_model,
	1424	+ 'status' => $status,
	1425	+ 'bot_id' => $bot_id,
	1426	+ ));
	1427	+ }
	1428	+
	1429	+ // Structured data so the Integrator's query-side adapter can rebuild its
	1430	+ // typed error contract (auth/rate-limit/quota/invalid-response) without a
	1431	+ // second transport implementation (876edb). Additive — message unchanged.
	1432	+ return new WP_Error('embedding_failed', $message, array(
	1433	+ 'branch' => 'cloud',
	1434	+ 'status' => $status,
	1435	+ 'error_type' => (is_array($decoded) && isset($decoded['error']['type']) && is_string($decoded['error']['type'])) ? $decoded['error']['type'] : '',
	1436	+ 'reason' => $reason,
	1437	+ 'model' => $selected_model,
	1438	+ ));
	1439	+}
	1440	+
	1441	+/**
	1442	+ * Generate an embedding via a Custom (OpenAI-compatible) provider's /embeddings route.
	1443	+ * Shared by every embedding entry point so the KNOWLEDGE-BASE INDEX side and the
	1444	+ * QUERY side route through the same model when the opt-in
	1445	+ * 'custom_provider_for_embeddings' setting is on. Mirrors the query-path logic in
	1446	+ * MxChat_Integrator::mxchat_generate_embedding_custom() but takes an explicit
	1447	+ * $options array so it is callable statically from utils + knowledge-manager.
	1448	+ *
	1449	+ * Returns a numeric array (the embedding vector) on success, or a human-readable
	1450	+ * error string on failure (so callers expecting a string error, like the
	1451	+ * knowledge-manager, can surface it directly; callers expecting array|null wrap it).
	1452	+ *
	1453	+ * @param string $text Text to embed.
	1454	+ * @param array $options The resolved mxchat options (must contain the custom_provider_* keys).
	1455	+ * @return array|string Embedding vector on success; error string on failure.
	1456	+ */
	1457	+public static function generate_embedding_custom($text, $options) {
	1458	+ if (empty($text)) {
	1459	+ return 'No text provided for embedding generation';
	1460	+ }
	1461	+
	1462	+ $base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
	1463	+ if (empty($base_url)) {
	1464	+ return 'Custom provider Base URL is not configured.';
	1465	+ }
	1466	+
	1467	+ $api_key = isset($options['custom_provider_api_key']) ? trim((string) $options['custom_provider_api_key']) : '';
	1468	+ $auth_scheme = isset($options['custom_provider_auth_scheme']) ? $options['custom_provider_auth_scheme'] : 'bearer';
	1469	+ $api_version = isset($options['custom_provider_api_version']) ? trim((string) $options['custom_provider_api_version']) : '';
	1470	+
	1471	+ // Embedding model: shared resolver (dedicated embedding model -> chat model
	1472	+ // -> 'default') — the mismatch warning's "selected" side reads the same chain.
	1473	+ $model = self::resolve_custom_embedding_model($options);
	1474	+
	1475	+ $embed_url = $base_url . '/embeddings';
	1476	+ if (!empty($api_version)) {
	1477	+ $embed_url .= (strpos($embed_url, '?') === false ? '?' : '&') . 'api-version=' . rawurlencode($api_version);
	1478	+ }
	1479	+
	1480	+ $headers = ['Content-Type' => 'application/json'];
	1481	+ if (!empty($api_key)) {
	1482	+ if ($auth_scheme === 'api-key') {
	1483	+ $headers['api-key'] = $api_key;
	1484	+ } else {
	1485	+ $headers['Authorization'] = 'Bearer ' . $api_key;
	1486	+ }
	1487	+ }
	1488	+
	1489	+ $response = wp_remote_post($embed_url, [
	1490	+ 'headers' => $headers,
	1491	+ 'body' => wp_json_encode(['input' => $text, 'model' => $model]),
	1492	+ 'timeout' => 60,
	1493	+ ]);
	1494	+ if (is_wp_error($response)) {
	1495	+ return self::log_custom_embedding_failure(
	1496	+ 'Connection error when generating embeddings (custom provider): ' . $response->get_error_message(),
	1497	+ $model,
	1498	+ $api_key
	1499	+ );
	1500	+ }
	1501	+
	1502	+ $status = wp_remote_retrieve_response_code($response);
	1503	+ $body = json_decode(wp_remote_retrieve_body($response), true);
	1504	+ if ($status !== 200) {
	1505	+ $msg = isset($body['error']['message']) ? $body['error']['message'] : 'HTTP ' . $status;
	1506	+ return self::log_custom_embedding_failure(
	1507	+ 'Custom embedding endpoint error: ' . $msg,
	1508	+ $model,
	1509	+ $api_key,
	1510	+ (int) $status
	1511	+ );
	1512	+ }
	1513	+ if (isset($body['data'][0]['embedding']) && is_array($body['data'][0]['embedding'])) {
	1514	+ // Stamp the custom model identity so the active-embedding-model mismatch
	1515	+ // warning reflects the real (custom) model rather than the built-in setting.
	1516	+ self::stamp_active_embedding_model('custom:' . $model);
	1517	+ return $body['data'][0]['embedding'];
	1518	+ }
	1519	+ return self::log_custom_embedding_failure('Invalid embedding response from custom provider.', $model, $api_key);
	1520	+}
	1521	+
	1522	+/**
	1523	+ * Record a custom-provider embedding failure in the Debug Mode log, then
	1524	+ * return the message unchanged so callers keep their string-error contract.
	1525	+ * The cloud branch has logged its failures since 4a7c0a; the custom branch
	1526	+ * never did, so chat-side failures on Custom-provider installs were
	1527	+ * invisible to Debug Mode despite the 3.2.18 readme saying otherwise
	1528	+ * (plan 71e4b6). Same scrub-then-log shape as embedding_failure_error().
	1529	+ *
	1530	+ * @param string $message Human-readable failure (the caller's return value).
	1531	+ * @param string $model Resolved custom embedding model.
	1532	+ * @param string $api_key Scrubbed out of the logged message if it ever appears.
	1533	+ * @param int $status HTTP status when one was received, 0 otherwise.
	1534	+ * @return string The (scrubbed) message.
	1535	+ */
	1536	+private static function log_custom_embedding_failure($message, $model, $api_key, $status = 0) {
	1537	+ if (is_string($api_key) && $api_key !== '') {
	1538	+ $message = str_replace($api_key, '[redacted]', $message);
	1539	+ }
	1540	+
	1541	+ if (class_exists('MxChat_Admin')) {
	1542	+ $context = array('model' => 'custom:' . $model);
	1543	+ if ($status > 0) {
	1544	+ $context['status'] = $status;
	1545	+ }
	1546	+ MxChat_Admin::mxchat_log_debug('embedding_error', $message, $context);
	1547	+ }
	1548	+
	1549	+ return $message;
	1550	+}
	1551	+
	1552	+/**
592	1553	* Submit content as multiple chunks
593	1554	*
594	1555	* Splits large content into chunks, generates embeddings for each,
595	1556	* and stores them with chunk metadata for later reassembly.
		@@ -608,10 +1569,26 @@
608	1569
609	1570	//error_log('[MXCHAT-CHUNK-DEBUG] Starting chunked submission for: ' . $source_url);
610	1571	//error_log('[MXCHAT-CHUNK-DEBUG] Content length: ' . strlen($content) . ' chars');
611	1572
612		- // First, delete any existing chunks for this URL (clean slate)
	1573	+ // URL-less (manual) content needs a minted identity BEFORE chunk ids are derived:
	1574	+ // every chunk id is md5(source_url)_chunk_N, so with source_url = '' EVERY long
	1575	+ // manual document shared the md5('') prefix — and the clean-slate delete below
	1576	+ // wiped the PREVIOUS manual entry's chunks each time a new one was added. The
	1577	+ // single-vector paths already mint (mxchat:// in WP, manual_* in Pinecone); this
	1578	+ // was the one storage path that didn't. Keep an mxchat:// identity if the caller
	1579	+ // already carries one.
	1580	+ if (strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, 'upload://') !== 0 && !preg_match('#^https?://#i', $source_url)) {
	1581	+ $source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
	1582	+ }
	1583	+
	1584	+ // First, delete any existing chunks for this URL (clean slate).
	1585	+ // Mirror-suspended: this is a re-store, not an entry removal — the Vector
	1586	+ // Store file is replaced (or kept, on hash match) by the caller's
	1587	+ // sync_upsert_entry after storage succeeds.
	1588	+ self::$vectorstore_mirror_suspended = true;
613	1589	$delete_result = self::delete_chunks_for_url($source_url, $bot_id);
	1590	+ self::$vectorstore_mirror_suspended = false;
614	1591	if (is_wp_error($delete_result)) {
615	1592	//error_log('[MXCHAT-CHUNK-DEBUG] Warning: Failed to delete existing chunks: ' . $delete_result->get_error_message());
616	1593	// Continue anyway - we'll overwrite with upsert
617	1594	}
		@@ -631,13 +1608,44 @@
631	1608	return new WP_Error('chunking_failed', 'Content could not be split into chunks');
632	1609	}
633	1610
634	1611	$errors = array();
	1612	+ $embed_failures = 0;
	1613	+ $first_embed_reason = '';
	1614	+ $first_store_reason = '';
635	1615	$is_pinecone = self::is_pinecone_enabled_for_bot($bot_id);
636	1616
637	1617	foreach ($chunks as $index => $chunk_text) {
638	1618	// Generate chunk metadata
639	1619	$chunk_metadata = MxChat_Chunker::create_chunk_metadata($index, $total_chunks, $source_url);
	1620	+
	1621	+ // AI-Engine-style aliases so external consumers (Pinecone/Qdrant/Chroma) can rely on
	1622	+ // a stable shorthand ('source'/'part_index'/'part_total') without parsing our internal names.
	1623	+ $chunk_metadata['source'] = $source_url;
	1624	+ $chunk_metadata['part_index'] = (int) $index;
	1625	+ $chunk_metadata['part_total'] = (int) $total_chunks;
	1626	+
	1627	+ /**
	1628	+ * Filter the per-chunk metadata blob before it's written to the KB store.
	1629	+ *
	1630	+ * @param array $chunk_metadata Metadata array (source, part_index, part_total, chunk_index, total_chunks, source_url, parent_url_hash, document_type, ...).
	1631	+ * @param string $chunk_text The chunk text being stored.
	1632	+ * @param array $context ['bot_id' => string, 'content_type' => string, 'source_url' => string, 'part_index' => int, 'part_total' => int]
	1633	+ * @return array Updated metadata array.
	1634	+ */
	1635	+ $chunk_metadata = apply_filters(
	1636	+ 'mxchat_embedding_chunk_metadata',
	1637	+ $chunk_metadata,
	1638	+ $chunk_text,
	1639	+ array(
	1640	+ 'bot_id' => $bot_id,
	1641	+ 'content_type' => $content_type,
	1642	+ 'source_url' => $source_url,
	1643	+ 'part_index' => (int) $index,
	1644	+ 'part_total' => (int) $total_chunks,
	1645	+ )
	1646	+ );
	1647	+
640	1648	$chunk_vector_id = MxChat_Chunker::generate_chunk_vector_id($source_url, $index);
641	1649
642	1650	//error_log('[MXCHAT-CHUNK] Processing chunk ' . ($index + 1) . '/' . $total_chunks . ' (ID: ' . $chunk_vector_id . ')');
643	1651
		@@ -644,10 +1652,17 @@
644	1652	// Generate embedding for this chunk
645	1653	$embedding_vector = self::generate_embedding($chunk_text, $api_key, $bot_id);
646	1654
647	1655	if (!is_array($embedding_vector)) {
648		- $errors[] = new WP_Error('embedding_failed', 'Failed to generate embedding for chunk ' . $index);
649		- //error_log('[MXCHAT-CHUNK] Failed to generate embedding for chunk ' . $index);
	1656	+ // Track embedding failures separately from storage failures, and
	1657	+ // keep the first provider reason seen — the two failure classes
	1658	+ // have opposite remedies (API key vs Pinecone/DB) (plan 4a7c0a).
	1659	+ $embed_failures++;
	1660	+ $reason = is_wp_error($embedding_vector) ? $embedding_vector->get_error_message() : '';
	1661	+ if ($reason !== '' && $first_embed_reason === '') {
	1662	+ $first_embed_reason = $reason;
	1663	+ }
	1664	+ $errors[] = new WP_Error('embedding_failed', 'Failed to generate embedding for chunk ' . $index . ($reason !== '' ? ' — ' . $reason : ''));
650	1665	continue;
651	1666	}
652	1667
653	1668	if ($is_pinecone) {
		@@ -677,20 +1692,44 @@
677	1692	}
678	1693
679	1694	if (is_wp_error($result)) {
680	1695	$errors[] = $result;
681		- //error_log('[MXCHAT-CHUNK] Failed to store chunk ' . $index . ': ' . $result->get_error_message());
	1696	+ if ($first_store_reason === '') {
	1697	+ $first_store_reason = $result->get_error_message();
	1698	+ }
682	1699	}
683	1700	}
684	1701
685	1702	if (count($errors) === $total_chunks) {
686		- return new WP_Error('chunking_failed', 'Failed to store any chunks');
	1703	+ // Say WHICH stage failed — "failed to store" used to cover pure
	1704	+ // embedding failures too, sending customers to debug Pinecone when
	1705	+ // the problem was their embedding API key (plan 4a7c0a).
	1706	+ if ($embed_failures === $total_chunks) {
	1707	+ return new WP_Error('chunking_failed',
	1708	+ 'Failed to store any chunks — every chunk failed to embed'
	1709	+ . ($first_embed_reason !== '' ? ': ' . $first_embed_reason : '')
	1710	+ . ' Check the embedding provider API key and model under MxChat Settings.');
	1711	+ }
	1712	+ if ($embed_failures === 0) {
	1713	+ return new WP_Error('chunking_failed',
	1714	+ 'Failed to store any chunks — embeddings generated but storage failed'
	1715	+ . ($first_store_reason !== '' ? ': ' . $first_store_reason : '')
	1716	+ . ' Check the knowledge base storage (Pinecone index or database).');
	1717	+ }
	1718	+ return new WP_Error('chunking_failed', sprintf(
	1719	+ 'Failed to store any chunks — %d failed to embed%s and %d failed to store%s',
	1720	+ $embed_failures,
	1721	+ $first_embed_reason !== '' ? ' (' . $first_embed_reason . ')' : '',
	1722	+ $total_chunks - $embed_failures,
	1723	+ $first_store_reason !== '' ? ' (' . $first_store_reason . ')' : ''
	1724	+ ));
687	1725	}
688	1726
689	1727	if (!empty($errors)) {
690		- //error_log('[MXCHAT-CHUNK] Completed with ' . count($errors) . ' errors out of ' . $total_chunks . ' chunks');
	1728	+ $detail = $first_embed_reason !== '' ? $first_embed_reason : $first_store_reason;
691	1729	return new WP_Error('chunking_partial_failure',
692		- sprintf('Failed to store %d of %d chunks', count($errors), $total_chunks));
	1730	+ sprintf('Failed to store %d of %d chunks', count($errors), $total_chunks)
	1731	+ . ($detail !== '' ? ' — first error: ' . $detail : ''));
693	1732	}
694	1733
695	1734	//error_log('[MXCHAT-CHUNK] Successfully stored all ' . $total_chunks . ' chunks');
696	1735	return true;
		@@ -737,11 +1776,23 @@
737	1776	'parent_url_hash' => $chunk_metadata['parent_url_hash'],
738	1777	'last_updated' => time(),
739	1778	'created_at' => time(),
740	1779	'bot_id' => $bot_id,
741		- 'embedding_model' => self::get_active_embedding_model()
742	1780	);
	1781	+ $page_title = self::source_title_for($source_url);
	1782	+ if ($page_title !== '') {
	1783	+ $metadata['title'] = $page_title; // page title captured at import (plan d9ee66)
	1784	+ }
743	1785
	1786	+ // Document index (plan 362c31): same chunk id and metadata, documents API.
	1787	+ if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) {
	1788	+ return MxChat_Pinecone_Documents::upsert_one($vector_id, $chunk_text, $embedding_vector, $metadata, array(
	1789	+ 'api_key' => $api_key,
	1790	+ 'host' => $host,
	1791	+ 'namespace' => $namespace,
	1792	+ ));
	1793	+ }
	1794	+
744	1795	$vector_data = array(
745	1796	'id' => $vector_id,
746	1797	'values' => $embedding_vector,
747	1798	'metadata' => $metadata
		@@ -784,21 +1835,25 @@
784	1835	global $wpdb;
785	1836
786	1837	// For chunks, we always insert new rows (no duplicate checking)
787	1838	// The URL includes chunk info in the metadata, but source_url stays the same for grouping
788		- $result = $wpdb->insert(
789		- $table_name,
790		- array(
791		- 'url' => $source_url,
792		- 'article_content' => $content_with_metadata,
793		- 'embedding_vector' => $embedding_vector_serialized,
794		- 'source_url' => $source_url,
795		- 'content_type' => $content_type,
796		- 'embedding_model' => self::get_active_embedding_model(),
797		- 'timestamp' => current_time('mysql')
798		- ),
799		- array('%s', '%s', '%s', '%s', '%s', '%s', '%s')
	1839	+ $row = array(
	1840	+ 'url' => $source_url,
	1841	+ 'article_content' => $content_with_metadata,
	1842	+ 'embedding_vector' => $embedding_vector_serialized,
	1843	+ 'source_url' => $source_url,
	1844	+ 'content_type' => $content_type,
	1845	+ 'timestamp' => current_time('mysql')
800	1846	);
	1847	+ $formats = array('%s', '%s', '%s', '%s', '%s', '%s');
	1848	+ // Every chunk row carries the page title too (plan d9ee66) - readers group by
	1849	+ // source_url and may pick any chunk as the representative.
	1850	+ $page_title = self::source_title_for($source_url);
	1851	+ if ($page_title !== '' && self::ensure_kb_title_column()) {
	1852	+ $row['title'] = $page_title;
	1853	+ $formats[] = '%s';
	1854	+ }
	1855	+ $result = $wpdb->insert($table_name, $row, $formats);
801	1856
802	1857	if ($result === false) {
803	1858	return new WP_Error('database_failed', 'Failed to insert chunk: ' . $wpdb->last_error);
804	1859	}
		@@ -815,8 +1870,14 @@
815	1870	*/
816	1871	public static function delete_chunks_for_url($source_url, $bot_id = 'default') {
817	1872	//error_log('[MXCHAT-CHUNK-DELETE] Deleting chunks for URL: ' . $source_url);
818	1873
	1874	+ // Entry removal — mirror it to the Vector Store (unless a storage routine
	1875	+ // is mid-re-store, see $vectorstore_mirror_suspended).
	1876	+ if (!self::$vectorstore_mirror_suspended && class_exists('MxChat_Vectorstore_Manager')) {
	1877	+ MxChat_Vectorstore_Manager::sync_delete_entry($source_url, $bot_id);
	1878	+ }
	1879	+
819	1880	if (self::is_pinecone_enabled_for_bot($bot_id)) {
820	1881	return self::delete_pinecone_chunks_by_url($source_url, $bot_id);
821	1882	} else {
822	1883	return self::delete_wordpress_chunks_by_url($source_url);
		@@ -825,10 +1886,16 @@
825	1886
826	1887	/**
827	1888	* Delete all chunks for a URL from Pinecone
828	1889	*/
829		-private static function delete_pinecone_chunks_by_url($source_url, $bot_id) {
830		- // Get Pinecone configuration
	1890	+/**
	1891	+ * Delete leftover md5(url)_chunk_N vectors after a URL's content was re-stored
	1892	+ * as a SINGLE vector (content shrank below the chunk threshold on edit/re-import).
	1893	+ * Unlike delete_pinecone_chunks_by_url this leaves the base id alone — the caller
	1894	+ * just upserted the new content there. Failure is logged, not fatal: the save
	1895	+ * itself succeeded, and the next save retries the sweep.
	1896	+ */
	1897	+private static function cleanup_pinecone_chunk_stragglers($source_url, $bot_id) {
831	1898	if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
832	1899	$pinecone_options = get_option('mxchat_pinecone_addon_options');
833	1900	$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
834	1901	$host = $pinecone_options['mxchat_pinecone_host'] ?? '';
		@@ -847,21 +1914,30 @@
847	1914	}
848	1915	}
849	1916
850	1917	if (empty($host) || empty($api_key)) {
851		- return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
	1918	+ return;
852	1919	}
853	1920
854		- $base_vector_id = md5($source_url);
855		- $vectors_to_delete = array();
	1921	+ // Document index (plan 362c31): list the chunk ids and delete them through the documents API.
	1922	+ if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) {
	1923	+ $docs_cfg = array('api_key' => $api_key, 'host' => $host, 'namespace' => $namespace);
	1924	+ $doc_stragglers = MxChat_Pinecone_Documents::list_ids(md5($source_url) . '_chunk_', $docs_cfg);
	1925	+ if (!empty($doc_stragglers)) {
	1926	+ $doc_result = MxChat_Pinecone_Documents::delete_ids($doc_stragglers, $docs_cfg);
	1927	+ if (is_wp_error($doc_result) && class_exists('MxChat_Admin') && method_exists('MxChat_Admin', 'mxchat_log_debug')) {
	1928	+ MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to sweep stale chunk documents after single-vector re-store', array('source_url' => $source_url, 'bot_id' => $bot_id, 'count' => count($doc_stragglers)));
	1929	+ }
	1930	+ }
	1931	+ return;
	1932	+ }
856	1933
857		- // Add the original single-vector ID (for non-chunked content)
858		- $vectors_to_delete[] = $base_vector_id;
	1934	+ $stragglers = array();
859	1935
860		- // Pinecone /vectors/list is a GET endpoint with query-string parameters; a POST here returns a
861		- // non-200 silently and we end up only deleting the base vector, leaving chunks orphaned.
	1936	+ // Pinecone /vectors/list is a GET endpoint with query-string parameters (a POST
	1937	+ // answers 200-with-an-empty-body, which reads as "no stragglers").
862	1938	$query_params = array(
863		- 'prefix' => $base_vector_id . '_chunk_',
	1939	+ 'prefix' => md5($source_url) . '_chunk_',
864	1940	'limit' => 100,
865	1941	);
866	1942	if (!empty($namespace)) {
867	1943	$query_params['namespace'] = $namespace;
		@@ -868,9 +1944,8 @@
868	1944	}
869	1945
870	1946	$list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
871	1947
872		- // Paginate in case a URL has more than 100 chunks.
873	1948	do {
874	1949	$list_response = wp_remote_get($list_url, array(
875	1950	'headers' => array(
876	1951	'Api-Key' => $api_key,
		@@ -886,9 +1961,9 @@
886	1961	$list_data = json_decode(wp_remote_retrieve_body($list_response), true);
887	1962	if (!empty($list_data['vectors'])) {
888	1963	foreach ($list_data['vectors'] as $vector) {
889	1964	if (isset($vector['id'])) {
890		- $vectors_to_delete[] = $vector['id'];
	1965	+ $stragglers[] = $vector['id'];
891	1966	}
892	1967	}
893	1968	}
894	1969
		@@ -900,8 +1975,188 @@
900	1975	$query_params['paginationToken'] = $next_token;
901	1976	$list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
902	1977	} while (true);
903	1978
	1979	+ if (empty($stragglers)) {
	1980	+ return;
	1981	+ }
	1982	+
	1983	+ $delete_body = array('ids' => $stragglers);
	1984	+ if (!empty($namespace)) {
	1985	+ $delete_body['namespace'] = $namespace;
	1986	+ }
	1987	+
	1988	+ $delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
	1989	+ 'headers' => array(
	1990	+ 'Api-Key' => $api_key,
	1991	+ 'accept' => 'application/json',
	1992	+ 'content-type' => 'application/json'
	1993	+ ),
	1994	+ 'body' => wp_json_encode($delete_body),
	1995	+ 'timeout' => 30
	1996	+ ));
	1997	+
	1998	+ if ((is_wp_error($delete_response) || wp_remote_retrieve_response_code($delete_response) !== 200)
	1999	+ && class_exists('MxChat_Admin') && method_exists('MxChat_Admin', 'mxchat_log_debug')) {
	2000	+ MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to sweep stale chunk vectors after single-vector re-store', array('source_url' => $source_url, 'bot_id' => $bot_id, 'count' => count($stragglers)));
	2001	+ }
	2002	+}
	2003	+
	2004	+/**
	2005	+ * Resolve the Pinecone credentials for a bot: the bot's own config when the
	2006	+ * multi-bot add-on supplies one, else the site-wide options. Extracted so the
	2007	+ * delete path and the count path can never resolve different credentials.
	2008	+ *
	2009	+ * @param string $bot_id
	2010	+ * @return array{api_key:string,host:string,namespace:string}
	2011	+ */
	2012	+private static function pinecone_config_for_bot($bot_id) {
	2013	+ if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
	2014	+ $bot_pinecone_config = array();
	2015	+ } else {
	2016	+ $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
	2017	+ }
	2018	+
	2019	+ if (empty($bot_pinecone_config)) {
	2020	+ $pinecone_options = get_option('mxchat_pinecone_addon_options');
	2021	+ return array(
	2022	+ 'api_key' => $pinecone_options['mxchat_pinecone_api_key'] ?? '',
	2023	+ 'host' => $pinecone_options['mxchat_pinecone_host'] ?? '',
	2024	+ 'namespace' => $pinecone_options['mxchat_pinecone_namespace'] ?? '',
	2025	+ // 'vector' | 'document' (plan 362c31) — the delete / count paths branch on it.
	2026	+ 'index_type' => (($pinecone_options['mxchat_pinecone_index_type'] ?? 'vector') === 'document') ? 'document' : 'vector',
	2027	+ );
	2028	+ }
	2029	+
	2030	+ return array(
	2031	+ 'api_key' => $bot_pinecone_config['api_key'] ?? '',
	2032	+ 'host' => $bot_pinecone_config['host'] ?? '',
	2033	+ 'namespace' => $bot_pinecone_config['namespace'] ?? '',
	2034	+ 'index_type' => (($bot_pinecone_config['index_type'] ?? 'vector') === 'document') ? 'document' : 'vector',
	2035	+ );
	2036	+}
	2037	+
	2038	+/**
	2039	+ * List every Pinecone vector id under a prefix, following pagination.
	2040	+ *
	2041	+ * Pinecone /vectors/list is a GET endpoint with query-string parameters; a POST here returns a
	2042	+ * non-200 silently and we end up seeing only the base vector, leaving chunks orphaned.
	2043	+ *
	2044	+ * @param string $prefix Vector-id prefix to match.
	2045	+ * @param array $cfg From pinecone_config_for_bot().
	2046	+ * @return array List of vector ids (empty when none match or the API is unreachable).
	2047	+ */
	2048	+private static function pinecone_list_ids_by_prefix($prefix, array $cfg) {
	2049	+ // Document index (plan 362c31): the documents API lists by prefix too.
	2050	+ if (($cfg['index_type'] ?? 'vector') === 'document' && class_exists('MxChat_Pinecone_Documents')) {
	2051	+ return MxChat_Pinecone_Documents::list_ids($prefix, $cfg);
	2052	+ }
	2053	+
	2054	+ $ids = array();
	2055	+
	2056	+ $query_params = array(
	2057	+ 'prefix' => $prefix,
	2058	+ 'limit' => 100,
	2059	+ );
	2060	+ if (!empty($cfg['namespace'])) {
	2061	+ $query_params['namespace'] = $cfg['namespace'];
	2062	+ }
	2063	+
	2064	+ $list_url = "https://{$cfg['host']}/vectors/list?" . http_build_query($query_params);
	2065	+
	2066	+ // Paginate in case a URL has more than 100 chunks.
	2067	+ do {
	2068	+ $list_response = wp_remote_get($list_url, array(
	2069	+ 'headers' => array(
	2070	+ 'Api-Key' => $cfg['api_key'],
	2071	+ 'accept' => 'application/json',
	2072	+ ),
	2073	+ 'timeout' => 30,
	2074	+ ));
	2075	+
	2076	+ if (is_wp_error($list_response) || wp_remote_retrieve_response_code($list_response) !== 200) {
	2077	+ break;
	2078	+ }
	2079	+
	2080	+ $list_data = json_decode(wp_remote_retrieve_body($list_response), true);
	2081	+ if (!empty($list_data['vectors'])) {
	2082	+ foreach ($list_data['vectors'] as $vector) {
	2083	+ if (isset($vector['id'])) {
	2084	+ $ids[] = $vector['id'];
	2085	+ }
	2086	+ }
	2087	+ }
	2088	+
	2089	+ $next_token = $list_data['pagination']['next'] ?? '';
	2090	+ if (empty($next_token)) {
	2091	+ break;
	2092	+ }
	2093	+
	2094	+ $query_params['paginationToken'] = $next_token;
	2095	+ $list_url = "https://{$cfg['host']}/vectors/list?" . http_build_query($query_params);
	2096	+ } while (true);
	2097	+
	2098	+ return $ids;
	2099	+}
	2100	+
	2101	+/**
	2102	+ * How many stored pieces a URL currently has, in whichever backend holds it.
	2103	+ *
	2104	+ * Branches on the SAME condition delete_chunks_for_url() branches on, so the
	2105	+ * number reported is the number that call removes. Used by the REST DELETE
	2106	+ * route to answer "was anything actually there?" without a second delete path.
	2107	+ *
	2108	+ * @param string $source_url
	2109	+ * @param string $bot_id
	2110	+ * @return int
	2111	+ */
	2112	+public static function count_chunks_for_url($source_url, $bot_id = 'default') {
	2113	+ if (self::is_pinecone_enabled_for_bot($bot_id)) {
	2114	+ $cfg = self::pinecone_config_for_bot($bot_id);
	2115	+ if (empty($cfg['host']) || empty($cfg['api_key'])) {
	2116	+ return 0;
	2117	+ }
	2118	+ // The base id and every "<base>_chunk_N" share the base as a prefix,
	2119	+ // so one prefix listing covers chunked and non-chunked content alike.
	2120	+ return count(self::pinecone_list_ids_by_prefix(md5($source_url), $cfg));
	2121	+ }
	2122	+
	2123	+ global $wpdb;
	2124	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	2125	+
	2126	+ return (int) $wpdb->get_var($wpdb->prepare(
	2127	+ "SELECT COUNT(*) FROM {$table_name} WHERE source_url = %s",
	2128	+ $source_url
	2129	+ ));
	2130	+}
	2131	+
	2132	+private static function delete_pinecone_chunks_by_url($source_url, $bot_id) {
	2133	+ $cfg = self::pinecone_config_for_bot($bot_id);
	2134	+ $api_key = $cfg['api_key'];
	2135	+ $host = $cfg['host'];
	2136	+ $namespace = $cfg['namespace'];
	2137	+
	2138	+ if (empty($host) || empty($api_key)) {
	2139	+ return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
	2140	+ }
	2141	+
	2142	+ // Document index (plan 362c31): base id + every <md5>_chunk_N, documents API.
	2143	+ if (($cfg['index_type'] ?? 'vector') === 'document' && class_exists('MxChat_Pinecone_Documents')) {
	2144	+ $doc_ids = array_merge(array(md5($source_url)), MxChat_Pinecone_Documents::list_ids(md5($source_url) . '_chunk_', $cfg));
	2145	+ return MxChat_Pinecone_Documents::delete_ids($doc_ids, $cfg);
	2146	+ }
	2147	+
	2148	+ $base_vector_id = md5($source_url);
	2149	+ $vectors_to_delete = array();
	2150	+
	2151	+ // Add the original single-vector ID (for non-chunked content)
	2152	+ $vectors_to_delete[] = $base_vector_id;
	2153	+
	2154	+ $vectors_to_delete = array_merge(
	2155	+ $vectors_to_delete,
	2156	+ self::pinecone_list_ids_by_prefix($base_vector_id . '_chunk_', $cfg)
	2157	+ );
	2158	+
904	2159	if (empty($vectors_to_delete)) {
905	2160	//error_log('[MXCHAT-CHUNK-DELETE] No vectors found to delete');
906	2161	return true;
907	2162	}
		@@ -960,6 +2215,297 @@
960	2215	}
961	2216
962	2217	//error_log('[MXCHAT-CHUNK-DELETE] Deleted ' . $result . ' rows from WordPress DB');
963	2218	return true;
	2219	+}
	2220	+
	2221	+/**
	2222	+ * Hybrid keyword boost (plan-38ffa1): detect whether the WP-DB knowledge
	2223	+ * table can serve the keyword leg via a MySQL FULLTEXT index, creating the
	2224	+ * index if needed. Detection runs once and caches the answer in the
	2225	+ * mxchat_hybrid_keyword_capability option ('fulltext' | 'like'); pass
	2226	+ * $force to re-detect. LIKE is the graceful fallback for shared hosts
	2227	+ * whose ALTER fails — the feature works either way, FULLTEXT just ranks
	2228	+ * better and scales.
	2229	+ *
	2230	+ * @param bool $force Re-run detection even if a cached answer exists.
	2231	+ * @return string 'fulltext' or 'like'
	2232	+ */
	2233	+public static function mxchat_hybrid_detect_capability($force = false) {
	2234	+ $cached = get_option('mxchat_hybrid_keyword_capability', '');
	2235	+ if (!$force && in_array($cached, array('fulltext', 'like'), true)) {
	2236	+ return $cached;
	2237	+ }
	2238	+
	2239	+ global $wpdb;
	2240	+ $table = $wpdb->prefix . 'mxchat_system_prompt_content';
	2241	+
	2242	+ $index_exists = $wpdb->get_var("SHOW INDEX FROM {$table} WHERE Key_name = 'mxchat_content_ft'");
	2243	+ if (!$index_exists) {
	2244	+ // Suppress the visible error on hosts where this is not permitted —
	2245	+ // failure is an expected, handled outcome (LIKE fallback).
	2246	+ $suppress = $wpdb->suppress_errors(true);
	2247	+ $wpdb->query("ALTER TABLE {$table} ADD FULLTEXT INDEX mxchat_content_ft (article_content)");
	2248	+ $wpdb->suppress_errors($suppress);
	2249	+ $index_exists = $wpdb->get_var("SHOW INDEX FROM {$table} WHERE Key_name = 'mxchat_content_ft'");
	2250	+ }
	2251	+
	2252	+ $capability = $index_exists ? 'fulltext' : 'like';
	2253	+ update_option('mxchat_hybrid_keyword_capability', $capability);
	2254	+ return $capability;
	2255	+}
	2256	+
	2257	+/**
	2258	+ * Public entry for re-embedding already-stored content in place (wp mxchat
	2259	+ * rtl-repair, plan d1e6f7). Thin wrapper so the repair CLI gets the exact
	2260	+ * provider routing the import path uses — the repaired vector must come from
	2261	+ * the same model family the bot indexes with, or retrieval stays broken.
	2262	+ */
	2263	+public static function regenerate_embedding($text, $api_key, $bot_id = 'default') {
	2264	+ return self::generate_embedding($text, $api_key, $bot_id);
	2265	+}
	2266	+
	2267	+/**
	2268	+ * Restore logical character order in PDF-extracted RTL text (plan 32bf9e).
	2269	+ *
	2270	+ * The bundled Smalot parser only un-reverses text runs tagged with the
	2271	+ * ReversedChars marked-content operator (Word emits it; LibreOffice and most
	2272	+ * other producers do not), so their Hebrew/Arabic PDFs extract in visual
	2273	+ * (reversed) order and embed/search as garbage. This is OUR post-processing
	2274	+ * seam over getText() — the parser itself is never patched (it gets replaced
	2275	+ * wholesale on library updates).
	2276	+ *
	2277	+ * Heuristic and deliberately conservative, per line:
	2278	+ * - lines without strong RTL codepoints are untouched (a fully-Latin line in
	2279	+ * an RTL document therefore stays as extracted — accepted limitation);
	2280	+ * - Arabic presentation forms are a definitive visual-order signal (they only
	2281	+ * appear in shaped output): de-shape to base letters and reverse;
	2282	+ * - otherwise flip only on positive evidence — Hebrew final-letter position
	2283	+ * (a sofit at word START only happens in reversed text) or sentence
	2284	+ * punctuation position (leading in visual order, trailing in logical);
	2285	+ * - ambiguous lines are left alone: a conservative miss beats corrupting a
	2286	+ * Word-produced extraction the parser already handled (the double-flip
	2287	+ * guard this plan's approval named mandatory).
	2288	+ *
	2289	+ * @param string $text One extracted page string, straight from getText().
	2290	+ * @param string $context Caller tag for the Debug Mode entry (site + page).
	2291	+ * @return string Text with RTL lines restored to logical order.
	2292	+ */
	2293	+public static function normalize_pdf_rtl($text, $context = '') {
	2294	+ if (!is_string($text) || '' === $text) {
	2295	+ return $text;
	2296	+ }
	2297	+ // Escape hatch for sites whose PDFs already extract logically.
	2298	+ if (!apply_filters('mxchat_pdf_rtl_normalize', true, $text)) {
	2299	+ return $text;
	2300	+ }
	2301	+ // Fast bail: nothing RTL anywhere in the page.
	2302	+ if (!preg_match('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $text)) {
	2303	+ return $text;
	2304	+ }
	2305	+
	2306	+ $parts = preg_split('/(\R)/u', $text, -1, PREG_SPLIT_DELIM_CAPTURE);
	2307	+ if (false === $parts) {
	2308	+ return $text;
	2309	+ }
	2310	+
	2311	+ $flipped_lines = 0;
	2312	+ $deshaped_lines = 0;
	2313	+ foreach ($parts as $i => $part) {
	2314	+ if ('' === $part || preg_match('/^\R$/u', $part)) {
	2315	+ continue;
	2316	+ }
	2317	+ $was_flipped = false;
	2318	+ $was_deshaped = false;
	2319	+ $new = self::pdf_rtl_normalize_line($part, $was_flipped, $was_deshaped);
	2320	+ if ($new !== $part) {
	2321	+ $parts[$i] = $new;
	2322	+ }
	2323	+ if ($was_flipped) {
	2324	+ $flipped_lines++;
	2325	+ }
	2326	+ if ($was_deshaped) {
	2327	+ $deshaped_lines++;
	2328	+ }
	2329	+ }
	2330	+
	2331	+ if (($flipped_lines || $deshaped_lines) && class_exists('MxChat_Admin')) {
	2332	+ MxChat_Admin::mxchat_log_debug('pdf_rtl_normalized', 'RTL PDF text restored to logical order', array(
	2333	+ 'context' => (string) $context,
	2334	+ 'lines_flipped' => $flipped_lines,
	2335	+ 'lines_deshaped' => $deshaped_lines,
	2336	+ 'decision' => 'visual-order extraction detected',
	2337	+ ));
	2338	+ }
	2339	+
	2340	+ return implode('', $parts);
	2341	+}
	2342	+
	2343	+/**
	2344	+ * Normalize one line. Sets $flipped/$deshaped for the caller's debug entry.
	2345	+ */
	2346	+private static function pdf_rtl_normalize_line($line, &$flipped, &$deshaped) {
	2347	+ $flipped = false;
	2348	+ $deshaped = false;
	2349	+
	2350	+ if (!preg_match('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $line)) {
	2351	+ return $line;
	2352	+ }
	2353	+
	2354	+ $has_forms = (bool) preg_match('/[\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $line);
	2355	+ $work = $line;
	2356	+ if ($has_forms) {
	2357	+ $work = strtr($work, self::pdf_rtl_deshape_map());
	2358	+ $deshaped = ($work !== $line);
	2359	+ }
	2360	+
	2361	+ $verdict = 'ambiguous';
	2362	+ if ($has_forms) {
	2363	+ // Shaped glyph codepoints only exist in visual-order output.
	2364	+ $verdict = 'visual';
	2365	+ } else {
	2366	+ // Strong-direction dominance gate first: an LTR-dominant line with an
	2367	+ // embedded RTL word is not flip material.
	2368	+ $rtl_count = preg_match_all('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}]/u', $work, $m_rtl);
	2369	+ $ltr_count = preg_match_all('/[A-Za-z]/u', $work, $m_ltr);
	2370	+ if ($rtl_count < 1 || $rtl_count <= $ltr_count) {
	2371	+ return $line;
	2372	+ }
	2373	+
	2374	+ // Hebrew final letters (ך ם ן ף ץ) end words in logical text; one at
	2375	+ // a word START (Hebrew letter follows, none precedes) is reversal
	2376	+ // evidence. Positional, so it survives the line being reversed.
	2377	+ $sofit_initial = preg_match_all('/(?<![\x{05D0}-\x{05EA}])[\x{05DA}\x{05DD}\x{05DF}\x{05E3}\x{05E5}](?=[\x{05D0}-\x{05EA}])/u', $work, $m_i);
	2378	+ $sofit_terminal = preg_match_all('/(?<=[\x{05D0}-\x{05EA}])[\x{05DA}\x{05DD}\x{05DF}\x{05E3}\x{05E5}](?![\x{05D0}-\x{05EA}])/u', $work, $m_t);
	2379	+ if ($sofit_initial > $sofit_terminal) {
	2380	+ $verdict = 'visual';
	2381	+ } elseif ($sofit_terminal > $sofit_initial) {
	2382	+ $verdict = 'logical';
	2383	+ } else {
	2384	+ // Sentence punctuation lands at the visual LEFT edge of an RTL
	2385	+ // line, i.e. the START of a visual-order extraction.
	2386	+ $trimmed = trim($work);
	2387	+ $starts_punct = (bool) preg_match('/^[.?!:;,]/u', $trimmed);
	2388	+ $ends_punct = (bool) preg_match('/[.?!:;,]$/u', $trimmed);
	2389	+ if ($starts_punct && !$ends_punct) {
	2390	+ $verdict = 'visual';
	2391	+ } elseif ($ends_punct && !$starts_punct) {
	2392	+ $verdict = 'logical';
	2393	+ }
	2394	+ }
	2395	+ }
	2396	+
	2397	+ if ('visual' !== $verdict) {
	2398	+ // Ambiguous or logical: hand back the original line UNLESS we
	2399	+ // de-shaped (de-shaping alone is always safe — same letters, same
	2400	+ // order, un-ligated).
	2401	+ return $deshaped ? $work : $line;
	2402	+ }
	2403	+
	2404	+ $flipped = true;
	2405	+ return self::pdf_rtl_flip_line($work);
	2406	+}
	2407	+
	2408	+/**
	2409	+ * Reverse a visual-order line into logical order: full character reversal,
	2410	+ * mirror paired punctuation, then re-reverse embedded LTR runs (Latin words
	2411	+ * and digit sequences, incl. Arabic-Indic digits) so they stay readable.
	2412	+ */
	2413	+private static function pdf_rtl_flip_line($line) {
	2414	+ $chars = preg_split('//u', $line, -1, PREG_SPLIT_NO_EMPTY);
	2415	+ if (false === $chars) {
	2416	+ return $line;
	2417	+ }
	2418	+ $reversed = implode('', array_reverse($chars));
	2419	+ $reversed = strtr($reversed, array(
	2420	+ '(' => ')', ')' => '(',
	2421	+ '[' => ']', ']' => '[',
	2422	+ '{' => '}', '}' => '{',
	2423	+ '<' => '>', '>' => '<',
	2424	+ ));
	2425	+ $restored = preg_replace_callback(
	2426	+ '/[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9}](?:[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9} .,\'"%\-:\/]*[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9}])?/u',
	2427	+ function ($m) {
	2428	+ $run = preg_split('//u', $m[0], -1, PREG_SPLIT_NO_EMPTY);
	2429	+ return false === $run ? $m[0] : implode('', array_reverse($run));
	2430	+ },
	2431	+ $reversed
	2432	+ );
	2433	+ return null === $restored ? $reversed : $restored;
	2434	+}
	2435	+
	2436	+/**
	2437	+ * Arabic presentation forms (A + B) -> base letters. Built once from range
	2438	+ * specs rather than ~120 hand-written literal entries; every codepoint in a
	2439	+ * range maps to the same base sequence (isolated/final/initial/medial forms
	2440	+ * of one letter are contiguous in the FE70 block).
	2441	+ */
	2442	+private static function pdf_rtl_deshape_map() {
	2443	+ static $map = null;
	2444	+ if (null !== $map) {
	2445	+ return $map;
	2446	+ }
	2447	+ $ranges = array(
	2448	+ // Form B harakat (each pair = standalone + tatweel-joined form).
	2449	+ array(0xFE70, 0xFE71, array(0x064B)), array(0xFE72, 0xFE72, array(0x064C)),
	2450	+ array(0xFE74, 0xFE74, array(0x064D)), array(0xFE76, 0xFE77, array(0x064E)),
	2451	+ array(0xFE78, 0xFE79, array(0x064F)), array(0xFE7A, 0xFE7B, array(0x0650)),
	2452	+ array(0xFE7C, 0xFE7D, array(0x0651)), array(0xFE7E, 0xFE7F, array(0x0652)),
	2453	+ // Form B letters.
	2454	+ array(0xFE80, 0xFE80, array(0x0621)), array(0xFE81, 0xFE82, array(0x0622)),
	2455	+ array(0xFE83, 0xFE84, array(0x0623)), array(0xFE85, 0xFE86, array(0x0624)),
	2456	+ array(0xFE87, 0xFE88, array(0x0625)), array(0xFE89, 0xFE8C, array(0x0626)),
	2457	+ array(0xFE8D, 0xFE8E, array(0x0627)), array(0xFE8F, 0xFE92, array(0x0628)),
	2458	+ array(0xFE93, 0xFE94, array(0x0629)), array(0xFE95, 0xFE98, array(0x062A)),
	2459	+ array(0xFE99, 0xFE9C, array(0x062B)), array(0xFE9D, 0xFEA0, array(0x062C)),
	2460	+ array(0xFEA1, 0xFEA4, array(0x062D)), array(0xFEA5, 0xFEA8, array(0x062E)),
	2461	+ array(0xFEA9, 0xFEAA, array(0x062F)), array(0xFEAB, 0xFEAC, array(0x0630)),
	2462	+ array(0xFEAD, 0xFEAE, array(0x0631)), array(0xFEAF, 0xFEB0, array(0x0632)),
	2463	+ array(0xFEB1, 0xFEB4, array(0x0633)), array(0xFEB5, 0xFEB8, array(0x0634)),
	2464	+ array(0xFEB9, 0xFEBC, array(0x0635)), array(0xFEBD, 0xFEC0, array(0x0636)),
	2465	+ array(0xFEC1, 0xFEC4, array(0x0637)), array(0xFEC5, 0xFEC8, array(0x0638)),
	2466	+ array(0xFEC9, 0xFECC, array(0x0639)), array(0xFECD, 0xFED0, array(0x063A)),
	2467	+ array(0xFED1, 0xFED4, array(0x0641)), array(0xFED5, 0xFED8, array(0x0642)),
	2468	+ array(0xFED9, 0xFEDC, array(0x0643)), array(0xFEDD, 0xFEE0, array(0x0644)),
	2469	+ array(0xFEE1, 0xFEE4, array(0x0645)), array(0xFEE5, 0xFEE8, array(0x0646)),
	2470	+ array(0xFEE9, 0xFEEC, array(0x0647)), array(0xFEED, 0xFEEE, array(0x0648)),
	2471	+ array(0xFEEF, 0xFEF0, array(0x0649)), array(0xFEF1, 0xFEF4, array(0x064A)),
	2472	+ // Form B lam-alef ligatures decompose to two letters.
	2473	+ array(0xFEF5, 0xFEF6, array(0x0644, 0x0622)), array(0xFEF7, 0xFEF8, array(0x0644, 0x0623)),
	2474	+ array(0xFEF9, 0xFEFA, array(0x0644, 0x0625)), array(0xFEFB, 0xFEFC, array(0x0644, 0x0627)),
	2475	+ // Form A: Persian / Urdu letters in common use.
	2476	+ array(0xFB56, 0xFB59, array(0x067E)), array(0xFB66, 0xFB69, array(0x0679)),
	2477	+ array(0xFB7A, 0xFB7D, array(0x0686)), array(0xFB88, 0xFB89, array(0x0688)),
	2478	+ array(0xFB8A, 0xFB8B, array(0x0698)), array(0xFB8E, 0xFB91, array(0x06A9)),
	2479	+ array(0xFB92, 0xFB95, array(0x06AF)), array(0xFBA6, 0xFBA9, array(0x06C1)),
	2480	+ array(0xFBAA, 0xFBAD, array(0x06BE)), array(0xFBAE, 0xFBAF, array(0x06D2)),
	2481	+ array(0xFBFC, 0xFBFF, array(0x06CC)),
	2482	+ );
	2483	+ $map = array();
	2484	+ foreach ($ranges as $range) {
	2485	+ $base = '';
	2486	+ foreach ($range[2] as $cp) {
	2487	+ $base .= self::pdf_rtl_cp_to_utf8($cp);
	2488	+ }
	2489	+ for ($cp = $range[0]; $cp <= $range[1]; $cp++) {
	2490	+ $map[self::pdf_rtl_cp_to_utf8($cp)] = $base;
	2491	+ }
	2492	+ }
	2493	+ return $map;
	2494	+}
	2495	+
	2496	+/**
	2497	+ * Codepoint to UTF-8 without ext-intl / mbstring entity tricks (PHP 7.2 floor).
	2498	+ */
	2499	+private static function pdf_rtl_cp_to_utf8($cp) {
	2500	+ if ($cp < 0x80) {
	2501	+ return chr($cp);
	2502	+ }
	2503	+ if ($cp < 0x800) {
	2504	+ return chr(0xC0 | ($cp >> 6)) . chr(0x80 | ($cp & 0x3F));
	2505	+ }
	2506	+ if ($cp < 0x10000) {
	2507	+ return chr(0xE0 | ($cp >> 12)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
	2508	+ }
	2509	+ return chr(0xF0 | ($cp >> 18)) . chr(0x80 | (($cp >> 12) & 0x3F)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
964	2510	}
965	2511	}