← All changes | includes/class-mxchat-utils.php +21 -107 3.2.7 → 3.1.5 View file →

		@@ -5,51 +5,8 @@
5	5
6	6	class MxChat_Utils {
7	7
8	8	/**
9		- * Centralized embedding model registry. Single source of truth for dimensions
10		- * and provider, so model-switch protection logic doesn't drift across files.
11		- */
12		-public static function embedding_model_registry() {
13		- return array(
14		- 'text-embedding-ada-002' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'Ada 2'),
15		- 'text-embedding-3-small' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'TE3 Small'),
16		- 'text-embedding-3-large' => array('dims' => 3072, 'provider' => 'openai', 'label' => 'TE3 Large'),
17		- 'voyage-3-large' => array('dims' => 2048, 'provider' => 'voyage', 'label' => 'Voyage-3 Large'),
18		- 'gemini-embedding-001' => array('dims' => 1536, 'provider' => 'gemini', 'label' => 'Gemini Embedding'),
19		- );
20		-}
21		-
22		-public static function embedding_model_dimensions($model) {
23		- $registry = self::embedding_model_registry();
24		- return isset($registry[$model]) ? (int) $registry[$model]['dims'] : 0;
25		-}
26		-
27		-public static function embedding_model_label($model) {
28		- $registry = self::embedding_model_registry();
29		- return isset($registry[$model]) ? $registry[$model]['label'] : $model;
30		-}
31		-
32		-/**
33		- * Returns the model that was last used to actually write embeddings into the
34		- * KB. Differs from the user-selected setting once a switch has happened but
35		- * no re-embed has occurred yet — that's the mismatch state we warn about.
36		- */
37		-public static function get_active_embedding_model() {
38		- return get_option('mxchat_active_embedding_model', '');
39		-}
40		-
41		-/**
42		- * Stamp the model that produced the most recent successful embedding. Called
43		- * from generate_embedding() right after the API responds with a valid vector.
44		- */
45		-public static function stamp_active_embedding_model($model) {
46		- if (!empty($model) && $model !== self::get_active_embedding_model()) {
47		- update_option('mxchat_active_embedding_model', $model, false);
48		- }
49		-}
50		-
51		-/**
52	9	* UPDATED: Submit or update content (and its embedding) in the database.
53	10	* Stores in Pinecone if enabled, otherwise stores in WordPress DB.
54	11	*
55	12	* @param string $content The content to be embedded.
		@@ -224,11 +181,9 @@
224	181	// Check if this is truly manual content (no URL at all) vs a real URL that filter_var rejects
225	182	// filter_var(FILTER_VALIDATE_URL) rejects valid URLs with encoded chars, non-ASCII, fragments, etc.
226	183	// Use a looser check: if it starts with http(s):// or has a scheme, it's a URL
227	184	$has_url_scheme = !empty($source_url) && preg_match('#^https?://#i', $source_url);
228		- // Treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
229		- $is_legacy_mxchat_url = $has_url_scheme && strpos($source_url, 'mxchat.ai') !== false;
230		- $is_manual_content = empty($source_url) || $source_url === '' || !$has_url_scheme || $is_legacy_mxchat_url;
	185	+ $is_manual_content = empty($source_url) || $source_url === '' || !$has_url_scheme;
231	186
232	187	if ($is_manual_content) {
233	188	// Generate unique identifier for manual content to prevent overwrites
234	189	$source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
		@@ -414,9 +369,9 @@
414	369	'source_url' => $url, // Can be empty for manual content
415	370	'type' => $content_type, // Now supports: post, page, pdf, url, manual, product, etc.
416	371	'last_updated' => time(),
417	372	'created_at' => time(), // Add creation timestamp
418		- 'bot_id' => $bot_id, // Add bot identification
	373	+ 'bot_id' => $bot_id // Add bot identification
419	374	);
420	375
421	376	$vector_data = array(
422	377	'id' => $vector_id,
		@@ -564,9 +519,8 @@
564	519	// Handle different response formats based on provider
565	520	if (strpos($selected_model, 'gemini-embedding') === 0) {
566	521	// Gemini API response format
567	522	if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
568		- self::stamp_active_embedding_model($selected_model);
569	523	return $response_body['embedding']['values'];
570	524	} else {
571	525	//error_log('Invalid response received from Gemini embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
572	526	return null;
		@@ -573,9 +527,8 @@
573	527	}
574	528	} else {
575	529	// OpenAI/Voyage API response format
576	530	if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
577		- self::stamp_active_embedding_model($selected_model);
578	531	return $response_body['data'][0]['embedding'];
579	532	} else {
580	533	//error_log('Invalid response received from embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
581	534	return null;
		@@ -631,36 +584,8 @@
631	584
632	585	foreach ($chunks as $index => $chunk_text) {
633	586	// Generate chunk metadata
634	587	$chunk_metadata = MxChat_Chunker::create_chunk_metadata($index, $total_chunks, $source_url);
635		-
636		- // AI-Engine-style aliases so external consumers (Pinecone/Qdrant/Chroma) can rely on
637		- // a stable shorthand ('source'/'part_index'/'part_total') without parsing our internal names.
638		- $chunk_metadata['source'] = $source_url;
639		- $chunk_metadata['part_index'] = (int) $index;
640		- $chunk_metadata['part_total'] = (int) $total_chunks;
641		-
642		- /**
643		- * Filter the per-chunk metadata blob before it's written to the KB store.
644		- *
645		- * @param array $chunk_metadata Metadata array (source, part_index, part_total, chunk_index, total_chunks, source_url, parent_url_hash, document_type, ...).
646		- * @param string $chunk_text The chunk text being stored.
647		- * @param array $context ['bot_id' => string, 'content_type' => string, 'source_url' => string, 'part_index' => int, 'part_total' => int]
648		- * @return array Updated metadata array.
649		- */
650		- $chunk_metadata = apply_filters(
651		- 'mxchat_embedding_chunk_metadata',
652		- $chunk_metadata,
653		- $chunk_text,
654		- array(
655		- 'bot_id' => $bot_id,
656		- 'content_type' => $content_type,
657		- 'source_url' => $source_url,
658		- 'part_index' => (int) $index,
659		- 'part_total' => (int) $total_chunks,
660		- )
661		- );
662		-
663	588	$chunk_vector_id = MxChat_Chunker::generate_chunk_vector_id($source_url, $index);
664	589
665	590	//error_log('[MXCHAT-CHUNK] Processing chunk ' . ($index + 1) . '/' . $total_chunks . ' (ID: ' . $chunk_vector_id . ')');
666	591
		@@ -759,9 +684,9 @@
759	684	'total_chunks' => $chunk_metadata['total_chunks'],
760	685	'parent_url_hash' => $chunk_metadata['parent_url_hash'],
761	686	'last_updated' => time(),
762	687	'created_at' => time(),
763		- 'bot_id' => $bot_id,
	688	+ 'bot_id' => $bot_id
764	689	);
765	690
766	691	$vector_data = array(
767	692	'id' => $vector_id,
		@@ -877,34 +802,31 @@
877	802
878	803	// Add the original single-vector ID (for non-chunked content)
879	804	$vectors_to_delete[] = $base_vector_id;
880	805
881		- // Pinecone /vectors/list is a GET endpoint with query-string parameters; a POST here returns a
882		- // non-200 silently and we end up only deleting the base vector, leaving chunks orphaned.
883		- $query_params = array(
	806	+ // Use Pinecone list API to find all chunk vectors with this prefix
	807	+ $list_url = "https://{$host}/vectors/list";
	808	+
	809	+ $list_body = array(
884	810	'prefix' => $base_vector_id . '_chunk_',
885		- 'limit' => 100,
	811	+ 'limit' => 100
886	812	);
	813	+
887	814	if (!empty($namespace)) {
888		- $query_params['namespace'] = $namespace;
	815	+ $list_body['namespace'] = $namespace;
889	816	}
890	817
891		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
	818	+ $list_response = wp_remote_post($list_url, array(
	819	+ 'headers' => array(
	820	+ 'Api-Key' => $api_key,
	821	+ 'accept' => 'application/json',
	822	+ 'content-type' => 'application/json'
	823	+ ),
	824	+ 'body' => wp_json_encode($list_body),
	825	+ 'timeout' => 30
	826	+ ));
892	827
893		- // Paginate in case a URL has more than 100 chunks.
894		- do {
895		- $list_response = wp_remote_get($list_url, array(
896		- 'headers' => array(
897		- 'Api-Key' => $api_key,
898		- 'accept' => 'application/json',
899		- ),
900		- 'timeout' => 30,
901		- ));
902		-
903		- if (is_wp_error($list_response) || wp_remote_retrieve_response_code($list_response) !== 200) {
904		- break;
905		- }
906		-
	828	+ if (!is_wp_error($list_response)) {
907	829	$list_data = json_decode(wp_remote_retrieve_body($list_response), true);
908	830	if (!empty($list_data['vectors'])) {
909	831	foreach ($list_data['vectors'] as $vector) {
910	832	if (isset($vector['id'])) {
		@@ -911,17 +833,9 @@
911	833	$vectors_to_delete[] = $vector['id'];
912	834	}
913	835	}
914	836	}
915		-
916		- $next_token = $list_data['pagination']['next'] ?? '';
917		- if (empty($next_token)) {
918		- break;
919		- }
920		-
921		- $query_params['paginationToken'] = $next_token;
922		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
923		- } while (true);
	837	+ }
924	838
925	839	if (empty($vectors_to_delete)) {
926	840	//error_log('[MXCHAT-CHUNK-DELETE] No vectors found to delete');
927	841	return true;