← All changes | includes/class-mxchat-utils.php +162 -81 2.0.3 → 2.1.3 View file →

		@@ -4,112 +4,193 @@
4	4	}
5	5
6	6	class MxChat_Utils {
7	7
8		- /**
9		- * Submit or update content (and its embedding) in the database.
10		- *
11		- * If the source_url already exists, update that row.
12		- * Otherwise, insert a new row.
13		- *
14		- * @param string $content The content to be embedded.
15		- * @param string $source_url The source URL of the content.
16		- * @param string $api_key The API key used for generating embeddings.
17		- */
18		- public static function submit_content_to_db($content, $source_url, $api_key) {
19		- global $wpdb;
20		- $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
21		-
22		- // Sanitize the source URL
23		- $source_url = esc_url_raw($source_url);
24		-
25		- // Generate the embedding using the API key
26		- $embedding_vector = self::generate_embedding($content, $api_key);
27		-
28		- if (is_array($embedding_vector)) {
29		- $embedding_vector_serialized = maybe_serialize($embedding_vector);
30		-
31		- // Check if a row already exists for this URL
32		- $existing_id = $wpdb->get_var(
33		- $wpdb->prepare(
34		- "SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
35		- $source_url
36		- )
37		- );
38		-
	8	+/**
	9	+ * Submit or update content (and its embedding) in the database.
	10	+ *
	11	+ * If the source_url already exists, update that row.
	12	+ * Otherwise, insert a new row.
	13	+ *
	14	+ * @param string $content The content to be embedded.
	15	+ * @param string $source_url The source URL of the content.
	16	+ * @param string $api_key The API key used for generating embeddings.
	17	+ * @return bool|string True on success, error message on failure
	18	+ */
	19	+public static function submit_content_to_db($content, $source_url, $api_key) {
	20	+ global $wpdb;
	21	+ $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
	22	+
	23	+ error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url);
	24	+ error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
	25	+
	26	+ // Sanitize the source URL
	27	+ $source_url = esc_url_raw($source_url);
	28	+
	29	+ // Use wpdb's prepare to ensure content is properly escaped for database
	30	+ $prepared_content = $wpdb->prepare('%s', $content);
	31	+ // Remove the quotes that wpdb->prepare adds
	32	+ $safe_content = substr($prepared_content, 1, -1);
	33	+
	34	+ // Additional content safety measures
	35	+ $safe_content = wp_check_invalid_utf8($safe_content);
	36	+ $safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
	37	+
	38	+ // Generate the embedding using the API key
	39	+ $embedding_vector = self::generate_embedding($content, $api_key);
	40	+
	41	+ if (!is_array($embedding_vector)) {
	42	+ error_log('[MXCHAT-DB] Error: Embedding generation failed');
	43	+ return false;
	44	+ }
	45	+
	46	+ error_log('[MXCHAT-DB] Embedding generated successfully');
	47	+
	48	+ $embedding_vector_serialized = maybe_serialize($embedding_vector);
	49	+ error_log('[MXCHAT-DB] Serialized embedding length: ' . strlen($embedding_vector_serialized) . ' bytes');
	50	+
	51	+ // Check if a row already exists for this URL
	52	+ $existing_id = $wpdb->get_var(
	53	+ $wpdb->prepare(
	54	+ "SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
	55	+ $source_url
	56	+ )
	57	+ );
	58	+
	59	+ // Progressive fallback mechanism for problematic content
	60	+ $attempt = 1;
	61	+ $max_attempts = 3;
	62	+ $current_content = $safe_content;
	63	+ $result = false;
	64	+
	65	+ while ($attempt <= $max_attempts && $result === false) {
	66	+ try {
39	67	if ($existing_id) {
	68	+ error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
	69	+
40	70	// Update the existing row
41		- $wpdb->update(
	71	+ $result = $wpdb->update(
42	72	$table_name,
43	73	array(
44		- 'article_content' => $content,
	74	+ 'url' => $source_url,
	75	+ 'article_content' => $current_content,
45	76	'embedding_vector' => $embedding_vector_serialized,
46		- 'timestamp' => current_time('mysql'), // Remove if your table doesn't have a timestamp column
	77	+ 'source_url' => $source_url,
	78	+ 'timestamp' => current_time('mysql'),
47	79	),
48	80	array('id' => $existing_id),
49		- array('%s','%s','%s'),
	81	+ array('%s','%s','%s','%s','%s'),
50	82	array('%d')
51	83	);
52	84	} else {
	85	+ error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
	86	+ error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
	87	+
53	88	// Insert a new row
54		- $wpdb->insert(
	89	+ $result = $wpdb->insert(
55	90	$table_name,
56	91	array(
57		- 'article_content' => $content,
	92	+ 'url' => $source_url,
	93	+ 'article_content' => $current_content,
58	94	'embedding_vector' => $embedding_vector_serialized,
59	95	'source_url' => $source_url,
60		- 'timestamp' => current_time('mysql'), // Remove if your table doesn't have a timestamp column
	96	+ 'timestamp' => current_time('mysql'),
61	97	),
62		- array('%s','%s','%s','%s')
	98	+ array('%s','%s','%s','%s','%s')
63	99	);
64	100	}
65		- } else {
66		- // If embedding generation failed, you could log or handle it here
67		- // error_log('Embedding generation failed for content from ' . $source_url);
	101	+
	102	+ if ($result === false) {
	103	+ error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . '): ' . $wpdb->last_error);
	104	+
	105	+ // Progressively apply more aggressive sanitization on failure
	106	+ if ($attempt === 1) {
	107	+ // First fallback: Use a more aggressive character filter and shorten
	108	+ $current_content = preg_replace('/[^\p{L}\p{N}\s.,;:!?()-]/u', '', $current_content);
	109	+ $current_content = substr($current_content, 0, 50000);
	110	+ } else if ($attempt === 2) {
	111	+ // Second fallback: Keep only alphanumeric and basic punctuation, shorten further
	112	+ $current_content = preg_replace('/[^a-zA-Z0-9\s.,;:!?()-]/u', '', $current_content);
	113	+ $current_content = substr($current_content, 0, 30000);
	114	+ }
	115	+
	116	+ $attempt++;
	117	+ }
	118	+ } catch (Exception $e) {
	119	+ error_log('[MXCHAT-DB] Exception during database operation: ' . $e->getMessage());
	120	+ $attempt++;
68	121	}
69	122	}
	123	+
	124	+ if ($result === false) {
	125	+ error_log('[MXCHAT-DB] All database operation attempts failed');
	126	+ return false;
	127	+ }
	128	+
	129	+ error_log('[MXCHAT-DB] Database operation completed successfully (Attempt ' . ($attempt - 1) . ')');
	130	+ return true;
	131	+}
70	132
71		- /**
72		- * Generate an embedding for the given text using the specified API key.
73		- *
74		- * @param string $text The text to be embedded.
75		- * @param string $api_key The API key used for generating embeddings.
76		- * @return array|null The embedding vector or null on failure.
77		- */
78		- private static function generate_embedding($text, $api_key) {
79		- $endpoint = 'https://api.openai.com/v1/embeddings';
	133	+/**
	134	+* Generate an embedding for the given text using the specified API key.
	135	+*
	136	+* @param string $text The text to be embedded.
	137	+* @param string $api_key The API key used for generating embeddings.
	138	+* @return array|null The embedding vector or null on failure.
	139	+*/
	140	+private static function generate_embedding($text, $api_key) {
	141	+ // Get options and selected model
	142	+ $options = get_option('mxchat_options');
	143	+ $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
	144	+
	145	+ // Determine endpoint and API key based on model
	146	+ if (strpos($selected_model, 'voyage') === 0) {
	147	+ $endpoint = 'https://api.voyageai.com/v1/embeddings';
	148	+ $api_key = $options['voyage_api_key'] ?? '';
	149	+ } else {
	150	+ $endpoint = 'https://api.openai.com/v1/embeddings';
	151	+ // Use the passed API key for OpenAI
	152	+ }
80	153
81		- $body = wp_json_encode([
82		- 'input' => $text,
83		- 'model' => 'text-embedding-ada-002'
84		- ]);
	154	+ // Prepare request body
	155	+ $request_body = [
	156	+ 'input' => $text,
	157	+ 'model' => $selected_model
	158	+ ];
85	159
86		- $args = [
87		- 'body' => $body,
88		- 'headers' => [
89		- 'Content-Type' => 'application/json',
90		- 'Authorization' => 'Bearer ' . $api_key,
91		- ],
92		- 'timeout' => 60,
93		- 'redirection' => 5,
94		- 'blocking' => true,
95		- 'httpversion' => '1.0',
96		- 'sslverify' => true,
97		- ];
	160	+ // Add output_dimension for voyage-3-large
	161	+ if ($selected_model === 'voyage-3-large') {
	162	+ $request_body['output_dimension'] = 2048;
	163	+ }
98	164
99		- $response = wp_remote_post($endpoint, $args);
	165	+ $args = [
	166	+ 'body' => wp_json_encode($request_body),
	167	+ 'headers' => [
	168	+ 'Content-Type' => 'application/json',
	169	+ 'Authorization' => 'Bearer ' . $api_key,
	170	+ ],
	171	+ 'timeout' => 60,
	172	+ 'redirection' => 5,
	173	+ 'blocking' => true,
	174	+ 'httpversion' => '1.0',
	175	+ 'sslverify' => true,
	176	+ ];
100	177
101		- if (is_wp_error($response)) {
102		- // error_log('Error generating embedding: ' . $response->get_error_message());
103		- return null;
104		- }
	178	+ $response = wp_remote_post($endpoint, $args);
	179	+
	180	+ if (is_wp_error($response)) {
	181	+ error_log('Error generating embedding: ' . $response->get_error_message());
	182	+ return null;
	183	+ }
105	184
106		- $response_body = json_decode(wp_remote_retrieve_body($response), true);
107		-
108		- if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
109		- return $response_body['data'][0]['embedding'];
110		- } else {
111		- // error_log('Invalid response received from embedding API.');
112		- return null;
113		- }
114		- }
	185	+ $response_body = json_decode(wp_remote_retrieve_body($response), true);
	186	+
	187	+ if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
	188	+ return $response_body['data'][0]['embedding'];
	189	+ } else {
	190	+ error_log('Invalid response received from embedding API: ' . wp_json_encode($response_body));
	191	+ return null;
	192	+ }
	193	+}
	194	+
	195	+
115	196	}