← All changes | includes/class-mxchat-utils.php +27 -199 3.2.13 → 3.1.4 View file →

		@@ -5,51 +5,8 @@
5	5
6	6	class MxChat_Utils {
7	7
8	8	/**
9		- * Centralized embedding model registry. Single source of truth for dimensions
10		- * and provider, so model-switch protection logic doesn't drift across files.
11		- */
12		-public static function embedding_model_registry() {
13		- return array(
14		- 'text-embedding-ada-002' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'Ada 2'),
15		- 'text-embedding-3-small' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'TE3 Small'),
16		- 'text-embedding-3-large' => array('dims' => 3072, 'provider' => 'openai', 'label' => 'TE3 Large'),
17		- 'voyage-3-large' => array('dims' => 2048, 'provider' => 'voyage', 'label' => 'Voyage-3 Large'),
18		- 'gemini-embedding-001' => array('dims' => 1536, 'provider' => 'gemini', 'label' => 'Gemini Embedding'),
19		- );
20		-}
21		-
22		-public static function embedding_model_dimensions($model) {
23		- $registry = self::embedding_model_registry();
24		- return isset($registry[$model]) ? (int) $registry[$model]['dims'] : 0;
25		-}
26		-
27		-public static function embedding_model_label($model) {
28		- $registry = self::embedding_model_registry();
29		- return isset($registry[$model]) ? $registry[$model]['label'] : $model;
30		-}
31		-
32		-/**
33		- * Returns the model that was last used to actually write embeddings into the
34		- * KB. Differs from the user-selected setting once a switch has happened but
35		- * no re-embed has occurred yet — that's the mismatch state we warn about.
36		- */
37		-public static function get_active_embedding_model() {
38		- return get_option('mxchat_active_embedding_model', '');
39		-}
40		-
41		-/**
42		- * Stamp the model that produced the most recent successful embedding. Called
43		- * from generate_embedding() right after the API responds with a valid vector.
44		- */
45		-public static function stamp_active_embedding_model($model) {
46		- if (!empty($model) && $model !== self::get_active_embedding_model()) {
47		- update_option('mxchat_active_embedding_model', $model, false);
48		- }
49		-}
50		-
51		-/**
52	9	* UPDATED: Submit or update content (and its embedding) in the database.
53	10	* Stores in Pinecone if enabled, otherwise stores in WordPress DB.
54	11	*
55	12	* @param string $content The content to be embedded.
		@@ -220,15 +177,9 @@
220	177	}
221	178
222	179	// ===== FIXED: Generate unique identifier for manual content =====
223	180	$original_source_url = $source_url;
224		- // Check if this is truly manual content (no URL at all) vs a real URL that filter_var rejects
225		- // filter_var(FILTER_VALIDATE_URL) rejects valid URLs with encoded chars, non-ASCII, fragments, etc.
226		- // Use a looser check: if it starts with http(s):// or has a scheme, it's a URL
227		- $has_url_scheme = !empty($source_url) && preg_match('#^https?://#i', $source_url);
228		- // Treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
229		- $is_legacy_mxchat_url = $has_url_scheme && strpos($source_url, 'mxchat.ai') !== false;
230		- $is_manual_content = empty($source_url) || $source_url === '' || !$has_url_scheme || $is_legacy_mxchat_url;
	181	+ $is_manual_content = empty($source_url) || $source_url === '' || !filter_var($source_url, FILTER_VALIDATE_URL);
231	182
232	183	if ($is_manual_content) {
233	184	// Generate unique identifier for manual content to prevent overwrites
234	185	$source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
		@@ -352,14 +303,14 @@
352	303	// ===== UPDATED: Handle manual content with unique vector IDs =====
353	304	if ($vector_id) {
354	305	// Use provided vector ID
355	306	//error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
356		- } elseif (!empty($url) && preg_match('#^https?://#i', $url)) {
357		- // For URLs, use URL-based ID (existing behavior)
	307	+ } elseif (!empty($url) && filter_var($url, FILTER_VALIDATE_URL)) {
	308	+ // For valid URLs, use URL-based ID (existing behavior)
358	309	$vector_id = md5($url);
359	310	//error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
360	311	} else {
361		- // For manual content (empty/no URL scheme), generate unique ID
	312	+ // For manual content (empty/invalid URL), generate unique ID
362	313	$vector_id = 'manual_' . time() . '_' . substr(md5($content . microtime(true)), 0, 8);
363	314	//error_log('[MXCHAT-PINECONE-MAIN] Generated unique vector ID for manual content: ' . $vector_id);
364	315	}
365	316	// ===== END UPDATE =====
		@@ -395,9 +346,9 @@
395	346	// Fallback to old detection logic for backwards compatibility
396	347	$is_product = false;
397	348	$content_type = 'manual'; // Default for manual content
398	349
399		- if (!empty($url) && preg_match('#^https?://#i', $url)) {
	350	+ if (!empty($url) && filter_var($url, FILTER_VALIDATE_URL)) {
400	351	$is_product = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
401	352	$content_type = $is_product ? 'product' : 'content';
402	353	}
403	354	}
		@@ -414,9 +365,9 @@
414	365	'source_url' => $url, // Can be empty for manual content
415	366	'type' => $content_type, // Now supports: post, page, pdf, url, manual, product, etc.
416	367	'last_updated' => time(),
417	368	'created_at' => time(), // Add creation timestamp
418		- 'bot_id' => $bot_id, // Add bot identification
	369	+ 'bot_id' => $bot_id // Add bot identification
419	370	);
420	371
421	372	$vector_data = array(
422	373	'id' => $vector_id,
		@@ -488,20 +439,11 @@
488	439	} else {
489	440	$bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
490	441	$options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
491	442	}
492		-
493		- // Opt-in: when the custom provider is selected for embeddings, route the KB
494		- // INDEX side through the same custom endpoint the query side uses, so stored
495		- // vectors and query vectors come from the same model. Default-off behavior
496		- // below is untouched.
497		- if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
498		- $custom = self::generate_embedding_custom($text, $options);
499		- return is_array($custom) ? $custom : null;
500		- }
501		-
	443	+
502	444	$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
503		-
	445	+
504	446	// Determine endpoint and API key based on model
505	447	if (strpos($selected_model, 'voyage') === 0) {
506	448	$endpoint = 'https://api.voyageai.com/v1/embeddings';
507	449	$api_key = $options['voyage_api_key'] ?? '';
		@@ -573,9 +515,8 @@
573	515	// Handle different response formats based on provider
574	516	if (strpos($selected_model, 'gemini-embedding') === 0) {
575	517	// Gemini API response format
576	518	if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
577		- self::stamp_active_embedding_model($selected_model);
578	519	return $response_body['embedding']['values'];
579	520	} else {
580	521	//error_log('Invalid response received from Gemini embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
581	522	return null;
		@@ -582,9 +523,8 @@
582	523	}
583	524	} else {
584	525	// OpenAI/Voyage API response format
585	526	if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
586		- self::stamp_active_embedding_model($selected_model);
587	527	return $response_body['data'][0]['embedding'];
588	528	} else {
589	529	//error_log('Invalid response received from embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
590	530	return null;
		@@ -592,81 +532,8 @@
592	532	}
593	533	}
594	534
595	535	/**
596		- * Generate an embedding via a Custom (OpenAI-compatible) provider's /embeddings route.
597		- * Shared by every embedding entry point so the KNOWLEDGE-BASE INDEX side and the
598		- * QUERY side route through the same model when the opt-in
599		- * 'custom_provider_for_embeddings' setting is on. Mirrors the query-path logic in
600		- * MxChat_Integrator::mxchat_generate_embedding_custom() but takes an explicit
601		- * $options array so it is callable statically from utils + knowledge-manager.
602		- *
603		- * Returns a numeric array (the embedding vector) on success, or a human-readable
604		- * error string on failure (so callers expecting a string error, like the
605		- * knowledge-manager, can surface it directly; callers expecting array|null wrap it).
606		- *
607		- * @param string $text Text to embed.
608		- * @param array $options The resolved mxchat options (must contain the custom_provider_* keys).
609		- * @return array|string Embedding vector on success; error string on failure.
610		- */
611		-public static function generate_embedding_custom($text, $options) {
612		- if (empty($text)) {
613		- return 'No text provided for embedding generation';
614		- }
615		-
616		- $base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
617		- if (empty($base_url)) {
618		- return 'Custom provider Base URL is not configured.';
619		- }
620		-
621		- $api_key = isset($options['custom_provider_api_key']) ? trim((string) $options['custom_provider_api_key']) : '';
622		- $auth_scheme = isset($options['custom_provider_auth_scheme']) ? $options['custom_provider_auth_scheme'] : 'bearer';
623		- $api_version = isset($options['custom_provider_api_version']) ? trim((string) $options['custom_provider_api_version']) : '';
624		-
625		- // Embedding model: prefer the dedicated custom_provider_embedding_model, fall back to the chat model.
626		- $model = (isset($options['custom_provider_embedding_model']) && trim((string) $options['custom_provider_embedding_model']) !== '')
627		- ? trim((string) $options['custom_provider_embedding_model'])
628		- : ((isset($options['custom_provider_model']) && trim((string) $options['custom_provider_model']) !== '') ? trim((string) $options['custom_provider_model']) : 'default');
629		-
630		- $embed_url = $base_url . '/embeddings';
631		- if (!empty($api_version)) {
632		- $embed_url .= (strpos($embed_url, '?') === false ? '?' : '&') . 'api-version=' . rawurlencode($api_version);
633		- }
634		-
635		- $headers = ['Content-Type' => 'application/json'];
636		- if (!empty($api_key)) {
637		- if ($auth_scheme === 'api-key') {
638		- $headers['api-key'] = $api_key;
639		- } else {
640		- $headers['Authorization'] = 'Bearer ' . $api_key;
641		- }
642		- }
643		-
644		- $response = wp_remote_post($embed_url, [
645		- 'headers' => $headers,
646		- 'body' => wp_json_encode(['input' => $text, 'model' => $model]),
647		- 'timeout' => 60,
648		- ]);
649		- if (is_wp_error($response)) {
650		- return 'Connection error when generating embeddings (custom provider): ' . $response->get_error_message();
651		- }
652		-
653		- $status = wp_remote_retrieve_response_code($response);
654		- $body = json_decode(wp_remote_retrieve_body($response), true);
655		- if ($status !== 200) {
656		- $msg = isset($body['error']['message']) ? $body['error']['message'] : 'HTTP ' . $status;
657		- return 'Custom embedding endpoint error: ' . $msg;
658		- }
659		- if (isset($body['data'][0]['embedding']) && is_array($body['data'][0]['embedding'])) {
660		- // Stamp the custom model identity so the active-embedding-model mismatch
661		- // warning reflects the real (custom) model rather than the built-in setting.
662		- self::stamp_active_embedding_model('custom:' . $model);
663		- return $body['data'][0]['embedding'];
664		- }
665		- return 'Invalid embedding response from custom provider.';
666		-}
667		-
668		-/**
669	536	* Submit content as multiple chunks
670	537	*
671	538	* Splits large content into chunks, generates embeddings for each,
672	539	* and stores them with chunk metadata for later reassembly.
		@@ -713,36 +580,8 @@
713	580
714	581	foreach ($chunks as $index => $chunk_text) {
715	582	// Generate chunk metadata
716	583	$chunk_metadata = MxChat_Chunker::create_chunk_metadata($index, $total_chunks, $source_url);
717		-
718		- // AI-Engine-style aliases so external consumers (Pinecone/Qdrant/Chroma) can rely on
719		- // a stable shorthand ('source'/'part_index'/'part_total') without parsing our internal names.
720		- $chunk_metadata['source'] = $source_url;
721		- $chunk_metadata['part_index'] = (int) $index;
722		- $chunk_metadata['part_total'] = (int) $total_chunks;
723		-
724		- /**
725		- * Filter the per-chunk metadata blob before it's written to the KB store.
726		- *
727		- * @param array $chunk_metadata Metadata array (source, part_index, part_total, chunk_index, total_chunks, source_url, parent_url_hash, document_type, ...).
728		- * @param string $chunk_text The chunk text being stored.
729		- * @param array $context ['bot_id' => string, 'content_type' => string, 'source_url' => string, 'part_index' => int, 'part_total' => int]
730		- * @return array Updated metadata array.
731		- */
732		- $chunk_metadata = apply_filters(
733		- 'mxchat_embedding_chunk_metadata',
734		- $chunk_metadata,
735		- $chunk_text,
736		- array(
737		- 'bot_id' => $bot_id,
738		- 'content_type' => $content_type,
739		- 'source_url' => $source_url,
740		- 'part_index' => (int) $index,
741		- 'part_total' => (int) $total_chunks,
742		- )
743		- );
744		-
745	584	$chunk_vector_id = MxChat_Chunker::generate_chunk_vector_id($source_url, $index);
746	585
747	586	//error_log('[MXCHAT-CHUNK] Processing chunk ' . ($index + 1) . '/' . $total_chunks . ' (ID: ' . $chunk_vector_id . ')');
748	587
		@@ -841,9 +680,9 @@
841	680	'total_chunks' => $chunk_metadata['total_chunks'],
842	681	'parent_url_hash' => $chunk_metadata['parent_url_hash'],
843	682	'last_updated' => time(),
844	683	'created_at' => time(),
845		- 'bot_id' => $bot_id,
	684	+ 'bot_id' => $bot_id
846	685	);
847	686
848	687	$vector_data = array(
849	688	'id' => $vector_id,
		@@ -959,34 +798,31 @@
959	798
960	799	// Add the original single-vector ID (for non-chunked content)
961	800	$vectors_to_delete[] = $base_vector_id;
962	801
963		- // Pinecone /vectors/list is a GET endpoint with query-string parameters; a POST here returns a
964		- // non-200 silently and we end up only deleting the base vector, leaving chunks orphaned.
965		- $query_params = array(
	802	+ // Use Pinecone list API to find all chunk vectors with this prefix
	803	+ $list_url = "https://{$host}/vectors/list";
	804	+
	805	+ $list_body = array(
966	806	'prefix' => $base_vector_id . '_chunk_',
967		- 'limit' => 100,
	807	+ 'limit' => 100
968	808	);
	809	+
969	810	if (!empty($namespace)) {
970		- $query_params['namespace'] = $namespace;
	811	+ $list_body['namespace'] = $namespace;
971	812	}
972	813
973		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
	814	+ $list_response = wp_remote_post($list_url, array(
	815	+ 'headers' => array(
	816	+ 'Api-Key' => $api_key,
	817	+ 'accept' => 'application/json',
	818	+ 'content-type' => 'application/json'
	819	+ ),
	820	+ 'body' => wp_json_encode($list_body),
	821	+ 'timeout' => 30
	822	+ ));
974	823
975		- // Paginate in case a URL has more than 100 chunks.
976		- do {
977		- $list_response = wp_remote_get($list_url, array(
978		- 'headers' => array(
979		- 'Api-Key' => $api_key,
980		- 'accept' => 'application/json',
981		- ),
982		- 'timeout' => 30,
983		- ));
984		-
985		- if (is_wp_error($list_response) || wp_remote_retrieve_response_code($list_response) !== 200) {
986		- break;
987		- }
988		-
	824	+ if (!is_wp_error($list_response)) {
989	825	$list_data = json_decode(wp_remote_retrieve_body($list_response), true);
990	826	if (!empty($list_data['vectors'])) {
991	827	foreach ($list_data['vectors'] as $vector) {
992	828	if (isset($vector['id'])) {
		@@ -993,17 +829,9 @@
993	829	$vectors_to_delete[] = $vector['id'];
994	830	}
995	831	}
996	832	}
997		-
998		- $next_token = $list_data['pagination']['next'] ?? '';
999		- if (empty($next_token)) {
1000		- break;
1001		- }
1002		-
1003		- $query_params['paginationToken'] = $next_token;
1004		- $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
1005		- } while (true);
	833	+ }
1006	834
1007	835	if (empty($vectors_to_delete)) {
1008	836	//error_log('[MXCHAT-CHUNK-DELETE] No vectors found to delete');
1009	837	return true;