← All changes | includes/class-mxchat-utils.php +367 -72 2.0.4 → 2.4.1 View file →

		@@ -4,112 +4,407 @@
4	4	}
5	5
6	6	class MxChat_Utils {
7	7
8		- /**
9		- * Submit or update content (and its embedding) in the database.
10		- *
11		- * If the source_url already exists, update that row.
12		- * Otherwise, insert a new row.
13		- *
14		- * @param string $content The content to be embedded.
15		- * @param string $source_url The source URL of the content.
16		- * @param string $api_key The API key used for generating embeddings.
17		- */
18		- public static function submit_content_to_db($content, $source_url, $api_key) {
19		- global $wpdb;
20		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	8	+/**
	9	+ * Submit or update content (and its embedding) in the database.
	10	+ * Stores in Pinecone if enabled, otherwise stores in WordPress DB.
	11	+ *
	12	+ * @param string $content The content to be embedded.
	13	+ * @param string $source_url The source URL of the content.
	14	+ * @param string $api_key The API key used for generating embeddings.
	15	+ * @param string $vector_id Optional vector ID for Pinecone (if not provided, will use md5 of URL)
	16	+ * @return bool|WP_Error True on success, WP_Error on failure
	17	+ */
	18	+public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null) {
	19	+ global $wpdb;
	20	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	21	+
	22	+ //error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url);
	23	+ //error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
	24	+
	25	+ // Sanitize the source URL
	26	+ $source_url = esc_url_raw($source_url);
	27	+
	28	+ // Just ensure UTF-8 validity without aggressive escaping
	29	+ $safe_content = wp_check_invalid_utf8($content);
	30	+ // Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D)
	31	+ $safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
21	32
22		- // Sanitize the source URL
23		- $source_url = esc_url_raw($source_url);
24	33
25		- // Generate the embedding using the API key
26		- $embedding_vector = self::generate_embedding($content, $api_key);
	34	+ // Generate the embedding using the API key
	35	+ $embedding_vector = self::generate_embedding($content, $api_key);
	36	+
	37	+ if (!is_array($embedding_vector)) {
	38	+ //error_log('[MXCHAT-DB] Error: Embedding generation failed');
	39	+ return new WP_Error('embedding_failed', 'Failed to generate embedding for content');
	40	+ }
	41	+
	42	+ //error_log('[MXCHAT-DB] Embedding generated successfully');
	43	+
	44	+ // Check if Pinecone is enabled and configured
	45	+ if (self::is_pinecone_enabled()) {
	46	+ //error_log('[MXCHAT-DB] Pinecone is enabled - using Pinecone storage');
	47	+ // Store in Pinecone only
	48	+ return self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id);
	49	+ } else {
	50	+ //error_log('[MXCHAT-DB] Pinecone not enabled - using WordPress storage');
	51	+ // Store in WordPress database only
	52	+ $embedding_vector_serialized = maybe_serialize($embedding_vector);
	53	+ return self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name);
	54	+ }
	55	+}
27	56
28		- if (is_array($embedding_vector)) {
29		- $embedding_vector_serialized = maybe_serialize($embedding_vector);
	57	+/**
	58	+ * Check if Pinecone is enabled and properly configured
	59	+ */
	60	+private static function is_pinecone_enabled() {
	61	+ $pinecone_options = get_option('mxchat_pinecone_addon_options');
	62	+
	63	+ if (empty($pinecone_options)) {
	64	+ return false;
	65	+ }
	66	+
	67	+ $enabled_check = !empty($pinecone_options['mxchat_use_pinecone']) && $pinecone_options['mxchat_use_pinecone'] !== '0';
	68	+ $api_key_check = !empty($pinecone_options['mxchat_pinecone_api_key']);
	69	+ $host_check = !empty($pinecone_options['mxchat_pinecone_host']);
	70	+
	71	+ return $enabled_check && $api_key_check && $host_check;
	72	+}
30	73
31		- // Check if a row already exists for this URL
32		- $existing_id = $wpdb->get_var(
33		- $wpdb->prepare(
34		- "SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
35		- $source_url
36		- )
37		- );
	74	+/**
	75	+ * Store content in Pinecone only
	76	+ */
	77	+private static function store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id = null) {
	78	+ //error_log('[MXCHAT-PINECONE] ===== Using Pinecone-only storage =====');
	79	+
	80	+ $pinecone_options = get_option('mxchat_pinecone_addon_options');
	81	+
	82	+ $result = self::store_in_pinecone_main(
	83	+ $embedding_vector,
	84	+ $content,
	85	+ $source_url,
	86	+ $pinecone_options['mxchat_pinecone_api_key'],
	87	+ $pinecone_options['mxchat_pinecone_environment'] ?? '',
	88	+ $pinecone_options['mxchat_pinecone_index'] ?? '',
	89	+ $vector_id
	90	+ );
	91	+
	92	+ if (is_wp_error($result)) {
	93	+ //error_log('[MXCHAT-PINECONE] Pinecone storage failed: ' . $result->get_error_message());
	94	+ return $result;
	95	+ }
	96	+
	97	+ //error_log('[MXCHAT-PINECONE] Pinecone storage completed successfully');
	98	+ return true;
	99	+}
38	100
	101	+/**
	102	+ * Store content in WordPress database with progressive fallback
	103	+ */
	104	+private static function store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name) {
	105	+ global $wpdb;
	106	+
	107	+ //error_log('[MXCHAT-DB] ===== Using WordPress-only storage =====');
	108	+
	109	+ // ===== FIXED: Generate unique identifier for manual content =====
	110	+ $original_source_url = $source_url;
	111	+ $is_manual_content = empty($source_url) || $source_url === '' || !filter_var($source_url, FILTER_VALIDATE_URL);
	112	+
	113	+ if ($is_manual_content) {
	114	+ // Generate unique identifier for manual content to prevent overwrites
	115	+ $source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
	116	+ //error_log('[MXCHAT-DB] Generated unique ID for manual content: ' . $source_url);
	117	+ }
	118	+
	119	+ // Only check for duplicates if we have a valid source URL (not manual content)
	120	+ $existing_id = null;
	121	+ if (!$is_manual_content) {
	122	+ $existing_id = $wpdb->get_var(
	123	+ $wpdb->prepare(
	124	+ "SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
	125	+ $source_url
	126	+ )
	127	+ );
	128	+ //error_log('[MXCHAT-DB] Checked for existing URL, found ID: ' . ($existing_id ?: 'none'));
	129	+ } else {
	130	+ //error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)');
	131	+ }
	132	+ // ===== END FIX =====
	133	+
	134	+ // Progressive fallback mechanism for problematic content
	135	+ $attempt = 1;
	136	+ $max_attempts = 3;
	137	+ $current_content = $safe_content;
	138	+ $result = false;
	139	+
	140	+ while ($attempt <= $max_attempts && $result === false) {
	141	+ try {
39	142	if ($existing_id) {
	143	+ //error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
	144	+
40	145	// Update the existing row
41		- $wpdb->update(
	146	+ $result = $wpdb->update(
42	147	$table_name,
43	148	array(
44		- 'article_content' => $content,
	149	+ 'url' => $source_url,
	150	+ 'article_content' => $current_content,
45	151	'embedding_vector' => $embedding_vector_serialized,
46		- 'timestamp' => current_time('mysql'), // Remove if your table doesn't have a timestamp column
	152	+ 'source_url' => $source_url,
	153	+ 'timestamp' => current_time('mysql'),
47	154	),
48	155	array('id' => $existing_id),
49		- array('%s','%s','%s'),
	156	+ array('%s','%s','%s','%s','%s'),
50	157	array('%d')
51	158	);
52	159	} else {
53		- // Insert a new row
54		- $wpdb->insert(
	160	+ //error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
	161	+ //error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
	162	+
	163	+ // Insert a new row (using generated unique ID for manual content)
	164	+ $result = $wpdb->insert(
55	165	$table_name,
56	166	array(
57		- 'article_content' => $content,
	167	+ 'url' => $source_url, // Now unique for manual content
	168	+ 'article_content' => $current_content,
58	169	'embedding_vector' => $embedding_vector_serialized,
59		- 'source_url' => $source_url,
60		- 'timestamp' => current_time('mysql'), // Remove if your table doesn't have a timestamp column
	170	+ 'source_url' => $source_url, // Now unique for manual content
	171	+ 'timestamp' => current_time('mysql'),
61	172	),
62		- array('%s','%s','%s','%s')
	173	+ array('%s','%s','%s','%s','%s')
63	174	);
64	175	}
65		- } else {
66		- // If embedding generation failed, you could log or handle it here
67		- // error_log('Embedding generation failed for content from ' . $source_url);
	176	+
	177	+ if ($result === false) {
	178	+ //error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . '): ' . $wpdb->last_error);
	179	+
	180	+ // Progressively apply more aggressive sanitization on failure
	181	+ if ($attempt === 1) {
	182	+ // First fallback: Use a more aggressive character filter and shorten
	183	+ $current_content = preg_replace('/[^\p{L}\p{N}\s.,;:!?()-]/u', '', $current_content);
	184	+ $current_content = substr($current_content, 0, 50000);
	185	+ } else if ($attempt === 2) {
	186	+ // Second fallback: Keep only alphanumeric and basic punctuation, shorten further
	187	+ $current_content = preg_replace('/[^a-zA-Z0-9\s.,;:!?()-]/u', '', $current_content);
	188	+ $current_content = substr($current_content, 0, 30000);
	189	+ }
	190	+
	191	+ $attempt++;
	192	+ }
	193	+ } catch (Exception $e) {
	194	+ //error_log('[MXCHAT-DB] Exception during database operation: ' . $e->getMessage());
	195	+ $attempt++;
68	196	}
69	197	}
	198	+
	199	+ if ($result === false) {
	200	+ //error_log('[MXCHAT-DB] All database operation attempts failed');
	201	+ return new WP_Error('database_failed', 'Failed to store content in WordPress database after ' . $max_attempts . ' attempts');
	202	+ }
	203	+
	204	+ //error_log('[MXCHAT-DB] WordPress database operation completed successfully (Attempt ' . ($attempt - 1) . ')');
	205	+ return true;
	206	+}
70	207
71		- /**
72		- * Generate an embedding for the given text using the specified API key.
73		- *
74		- * @param string $text The text to be embedded.
75		- * @param string $api_key The API key used for generating embeddings.
76		- * @return array|null The embedding vector or null on failure.
77		- */
78		- private static function generate_embedding($text, $api_key) {
79		- $endpoint = 'https://api.openai.com/v1/embeddings';
	208	+/**
	209	+ * Store content in Pinecone database
	210	+ */
	211	+private static function store_in_pinecone_main($embedding_vector, $content, $url, $api_key, $environment, $index_name, $vector_id = null) {
	212	+ //error_log('[MXCHAT-PINECONE-MAIN] ===== Starting Pinecone storage =====');
	213	+
	214	+ // ===== UPDATED: Handle manual content with unique vector IDs =====
	215	+ if ($vector_id) {
	216	+ // Use provided vector ID
	217	+ //error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
	218	+ } elseif (!empty($url) && filter_var($url, FILTER_VALIDATE_URL)) {
	219	+ // For valid URLs, use URL-based ID (existing behavior)
	220	+ $vector_id = md5($url);
	221	+ //error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
	222	+ } else {
	223	+ // For manual content (empty/invalid URL), generate unique ID
	224	+ $vector_id = 'manual_' . time() . '_' . substr(md5($content . microtime(true)), 0, 8);
	225	+ //error_log('[MXCHAT-PINECONE-MAIN] Generated unique vector ID for manual content: ' . $vector_id);
	226	+ }
	227	+ // ===== END UPDATE =====
	228	+
	229	+ $options = get_option('mxchat_pinecone_addon_options');
	230	+ $host = $options['mxchat_pinecone_host'] ?? '';
	231	+
	232	+ //error_log('[MXCHAT-PINECONE-MAIN] Host from options: ' . $host);
	233	+ //error_log('[MXCHAT-PINECONE-MAIN] API key length: ' . strlen($api_key));
	234	+ //error_log('[MXCHAT-PINECONE-MAIN] Environment: ' . $environment);
	235	+ //error_log('[MXCHAT-PINECONE-MAIN] Index name: ' . $index_name);
80	236
81		- $body = wp_json_encode([
82		- 'input' => $text,
83		- 'model' => 'text-embedding-ada-002'
84		- ]);
	237	+ if (empty($host)) {
	238	+ //error_log('[MXCHAT-PINECONE-MAIN] ERROR: Host is empty');
	239	+ return new WP_Error('pinecone_config', 'Pinecone host is not configured. Please set the host in your settings.');
	240	+ }
85	241
86		- $args = [
87		- 'body' => $body,
88		- 'headers' => [
89		- 'Content-Type' => 'application/json',
90		- 'Authorization' => 'Bearer ' . $api_key,
	242	+ // ===== UPDATED: Determine content type more accurately =====
	243	+ $is_product = false;
	244	+ $content_type = 'manual'; // Default for manual content
	245	+
	246	+ if (!empty($url) && filter_var($url, FILTER_VALIDATE_URL)) {
	247	+ $is_product = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
	248	+ $content_type = $is_product ? 'product' : 'content';
	249	+ }
	250	+
	251	+ //error_log('[MXCHAT-PINECONE-MAIN] Content type: ' . $content_type);
	252	+ // ===== END UPDATE =====
	253	+
	254	+ $api_endpoint = "https://{$host}/vectors/upsert";
	255	+ //error_log('[MXCHAT-PINECONE-MAIN] API endpoint: ' . $api_endpoint);
	256	+
	257	+ $request_body = array(
	258	+ 'vectors' => array(
	259	+ array(
	260	+ 'id' => $vector_id,
	261	+ 'values' => $embedding_vector,
	262	+ 'metadata' => array(
	263	+ 'text' => $content,
	264	+ 'source_url' => $url, // Can be empty for manual content
	265	+ 'type' => $content_type, // 'manual', 'content', or 'product'
	266	+ 'last_updated' => time(),
	267	+ 'created_at' => time() // Add creation timestamp
	268	+ )
	269	+ )
	270	+ )
	271	+ );
	272	+
	273	+ //error_log('[MXCHAT-PINECONE-MAIN] Request body prepared (embedding dimensions: ' . count($embedding_vector) . ')');
	274	+
	275	+ $response = wp_remote_post($api_endpoint, array(
	276	+ 'headers' => array(
	277	+ 'Api-Key' => $api_key,
	278	+ 'accept' => 'application/json',
	279	+ 'content-type' => 'application/json'
	280	+ ),
	281	+ 'body' => wp_json_encode($request_body),
	282	+ 'timeout' => 30,
	283	+ 'data_format' => 'body'
	284	+ ));
	285	+
	286	+ if (is_wp_error($response)) {
	287	+ //error_log('[MXCHAT-PINECONE-MAIN] WordPress request error: ' . $response->get_error_message());
	288	+ return new WP_Error('pinecone_request', $response->get_error_message());
	289	+ }
	290	+
	291	+ $response_code = wp_remote_retrieve_response_code($response);
	292	+ //error_log('[MXCHAT-PINECONE-MAIN] Response code: ' . $response_code);
	293	+
	294	+ if ($response_code !== 200) {
	295	+ $body = wp_remote_retrieve_body($response);
	296	+ //error_log('[MXCHAT-PINECONE-MAIN] API error - Response body: ' . $body);
	297	+ return new WP_Error('pinecone_api', sprintf(
	298	+ 'Pinecone API error (HTTP %d): %s',
	299	+ $response_code,
	300	+ $body
	301	+ ));
	302	+ }
	303	+
	304	+ $response_body = wp_remote_retrieve_body($response);
	305	+ //error_log('[MXCHAT-PINECONE-MAIN] Success response: ' . $response_body);
	306	+ //error_log('[MXCHAT-PINECONE-MAIN] Successfully stored in Pinecone');
	307	+ //error_log('[MXCHAT-PINECONE-MAIN] ===== Pinecone storage complete =====');
	308	+
	309	+ return true;
	310	+}
	311	+/**
	312	+ * Generate an embedding for the given text using the specified API key.
	313	+ *
	314	+ * @param string $text The text to be embedded.
	315	+ * @param string $api_key The API key used for generating embeddings.
	316	+ * @return array|null The embedding vector or null on failure.
	317	+ */
	318	+private static function generate_embedding($text, $api_key) {
	319	+ // Get options and selected model
	320	+ $options = get_option('mxchat_options');
	321	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	322	+
	323	+ // Determine endpoint and API key based on model
	324	+ if (strpos($selected_model, 'voyage') === 0) {
	325	+ $endpoint = 'https://api.voyageai.com/v1/embeddings';
	326	+ $api_key = $options['voyage_api_key'] ?? '';
	327	+ } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
	328	+ $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
	329	+ $api_key = $options['gemini_api_key'] ?? '';
	330	+ } else {
	331	+ $endpoint = 'https://api.openai.com/v1/embeddings';
	332	+ // Use the passed API key for OpenAI
	333	+ }
	334	+
	335	+ // Prepare request body based on provider
	336	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	337	+ // Gemini API format
	338	+ $request_body = [
	339	+ 'model' => 'models/' . $selected_model,
	340	+ 'content' => [
	341	+ 'parts' => [
	342	+ ['text' => $text]
	343	+ ]
91	344	],
92		- 'timeout' => 60,
93		- 'redirection' => 5,
94		- 'blocking' => true,
95		- 'httpversion' => '1.0',
96		- 'sslverify' => true,
	345	+ 'outputDimensionality' => 1536
97	346	];
98		-
99		- $response = wp_remote_post($endpoint, $args);
100		-
101		- if (is_wp_error($response)) {
102		- // error_log('Error generating embedding: ' . $response->get_error_message());
	347	+
	348	+ // Prepare headers for Gemini (API key as query parameter)
	349	+ $endpoint .= '?key=' . $api_key;
	350	+ $headers = [
	351	+ 'Content-Type' => 'application/json'
	352	+ ];
	353	+ } else {
	354	+ // OpenAI/Voyage API format
	355	+ $request_body = [
	356	+ 'input' => $text,
	357	+ 'model' => $selected_model
	358	+ ];
	359	+
	360	+ // Add output_dimension for voyage-3-large
	361	+ if ($selected_model === 'voyage-3-large') {
	362	+ $request_body['output_dimension'] = 2048;
	363	+ }
	364	+
	365	+ // Prepare headers for OpenAI/Voyage
	366	+ $headers = [
	367	+ 'Content-Type' => 'application/json',
	368	+ 'Authorization' => 'Bearer ' . $api_key
	369	+ ];
	370	+ }
	371	+
	372	+ $args = [
	373	+ 'body' => wp_json_encode($request_body),
	374	+ 'headers' => $headers,
	375	+ 'timeout' => 60,
	376	+ 'redirection' => 5,
	377	+ 'blocking' => true,
	378	+ 'httpversion' => '1.0',
	379	+ 'sslverify' => true,
	380	+ ];
	381	+
	382	+ $response = wp_remote_post($endpoint, $args);
	383	+
	384	+ if (is_wp_error($response)) {
	385	+ //error_log('Error generating embedding: ' . $response->get_error_message());
	386	+ return null;
	387	+ }
	388	+
	389	+ $response_body = json_decode(wp_remote_retrieve_body($response), true);
	390	+
	391	+ // Handle different response formats based on provider
	392	+ if (strpos($selected_model, 'gemini-embedding') === 0) {
	393	+ // Gemini API response format
	394	+ if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
	395	+ return $response_body['embedding']['values'];
	396	+ } else {
	397	+ //error_log('Invalid response received from Gemini embedding API: ' . wp_json_encode($response_body));
103	398	return null;
104	399	}
105		-
106		- $response_body = json_decode(wp_remote_retrieve_body($response), true);
107		-
	400	+ } else {
	401	+ // OpenAI/Voyage API response format
108	402	if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
109	403	return $response_body['data'][0]['embedding'];
110	404	} else {
111		- // error_log('Invalid response received from embedding API.');
	405	+ //error_log('Invalid response received from embedding API: ' . wp_json_encode($response_body));
112	406	return null;
113	407	}
114	408	}
115		-}
	409	+}
	410	+}