PluginProbe
MxChat – AI Chatbot & Content Generation for WordPress / 2.4.6
MxChat – AI Chatbot & Content Generation for WordPress v2.4.6
3.2.21 3.2.20 3.2.19 3.2.18 3.2.17 3.2.16 3.2.15 3.2.14 3.2.12 3.2.13 3.2.11 3.2.10 3.2.9 3.2.8 3.2.7 3.2.6 3.2.5 3.2.4 3.2.3 3.2.2 3.2.1 2.0.3 2.0.4 2.0.5 2.0.6 All 152 releases
← All changes | includes/class-mxchat-utils.php +487 -1067 3.2.11 → 2.4.6 View file →
@@ -1,1068 +1,488 @@
1 -<?php
2 -if (!defined('ABSPATH')) {
3 - exit; // Exit if accessed directly
4 -}
5 -
6 -class MxChat_Utils {
7 -
8 -/**
9 - * Centralized embedding model registry. Single source of truth for dimensions
10 - * and provider, so model-switch protection logic doesn't drift across files.
11 - */
12 -public static function embedding_model_registry() {
13 - return array(
14 - 'text-embedding-ada-002' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'Ada 2'),
15 - 'text-embedding-3-small' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'TE3 Small'),
16 - 'text-embedding-3-large' => array('dims' => 3072, 'provider' => 'openai', 'label' => 'TE3 Large'),
17 - 'voyage-3-large' => array('dims' => 2048, 'provider' => 'voyage', 'label' => 'Voyage-3 Large'),
18 - 'gemini-embedding-001' => array('dims' => 1536, 'provider' => 'gemini', 'label' => 'Gemini Embedding'),
19 - );
20 -}
21 -
22 -public static function embedding_model_dimensions($model) {
23 - $registry = self::embedding_model_registry();
24 - return isset($registry[$model]) ? (int) $registry[$model]['dims'] : 0;
25 -}
26 -
27 -public static function embedding_model_label($model) {
28 - $registry = self::embedding_model_registry();
29 - return isset($registry[$model]) ? $registry[$model]['label'] : $model;
30 -}
31 -
32 -/**
33 - * Returns the model that was last used to actually write embeddings into the
34 - * KB. Differs from the user-selected setting once a switch has happened but
35 - * no re-embed has occurred yet — that's the mismatch state we warn about.
36 - */
37 -public static function get_active_embedding_model() {
38 - return get_option('mxchat_active_embedding_model', '');
39 -}
40 -
41 -/**
42 - * Stamp the model that produced the most recent successful embedding. Called
43 - * from generate_embedding() right after the API responds with a valid vector.
44 - */
45 -public static function stamp_active_embedding_model($model) {
46 - if (!empty($model) && $model !== self::get_active_embedding_model()) {
47 - update_option('mxchat_active_embedding_model', $model, false);
48 - }
49 -}
50 -
51 -/**
52 - * UPDATED: Submit or update content (and its embedding) in the database.
53 - * Stores in Pinecone if enabled, otherwise stores in WordPress DB.
54 - *
55 - * @param string $content The content to be embedded.
56 - * @param string $source_url The source URL of the content.
57 - * @param string $api_key The API key used for generating embeddings.
58 - * @param string $vector_id Optional vector ID for Pinecone (if not provided, will use md5 of URL)
59 - * @param string $bot_id The bot ID for multi-bot support
60 - * @param string $content_type The type of content (post, page, pdf, url, manual, product, etc.)
61 - * @return bool|WP_Error True on success, WP_Error on failure
62 - */
63 -public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
64 - global $wpdb;
65 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
66 -
67 - //error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ', Type: ' . $content_type . ')');
68 - //error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
69 -
70 - // Sanitize the source URL
71 - $source_url = esc_url_raw($source_url);
72 -
73 - // Sanitize content_type
74 - $content_type = sanitize_key($content_type);
75 - if (empty($content_type)) {
76 - $content_type = 'content'; // Fallback for backwards compatibility
77 - }
78 -
79 - // Just ensure UTF-8 validity without aggressive escaping
80 - $safe_content = wp_check_invalid_utf8($content);
81 - // Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D)
82 - $safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
83 -
84 - // Check if chunking should be applied
85 - $chunker = MxChat_Chunker::from_settings();
86 - if ($chunker->should_chunk($safe_content)) {
87 - //error_log('[MXCHAT-DB] Content exceeds chunk threshold, using chunked submission');
88 - return self::submit_chunked_content($safe_content, $source_url, $api_key, $bot_id, $content_type, $chunker);
89 - }
90 -
91 - // UPDATED: Generate the embedding using bot-specific configuration
92 - $embedding_vector = self::generate_embedding($content, $api_key, $bot_id);
93 -
94 - if (!is_array($embedding_vector)) {
95 - //error_log('[MXCHAT-DB] Error: Embedding generation failed');
96 - return new WP_Error('embedding_failed', 'Failed to generate embedding for content');
97 - }
98 -
99 - //error_log('[MXCHAT-DB] Embedding generated successfully');
100 -
101 - // UPDATED: Check if Pinecone is enabled for this specific bot
102 - if (self::is_pinecone_enabled_for_bot($bot_id)) {
103 - //error_log('[MXCHAT-DB] Pinecone is enabled for bot ' . $bot_id . ' - using Pinecone storage');
104 - // Store in Pinecone only
105 - return self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id, $content_type);
106 - } else {
107 - //error_log('[MXCHAT-DB] Pinecone not enabled for bot ' . $bot_id . ' - using WordPress storage');
108 - // Store in WordPress database only
109 - $embedding_vector_serialized = maybe_serialize($embedding_vector);
110 - return self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type);
111 - }
112 -}
113 -
114 -/**
115 - * UPDATED: Check if Pinecone is enabled and properly configured for a specific bot
116 - */
117 -private static function is_pinecone_enabled_for_bot($bot_id = 'default') {
118 - // For default bot or when multi-bot is not active, use original method
119 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
120 - return self::is_pinecone_enabled();
121 - }
122 -
123 - // Get bot-specific Pinecone configuration
124 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
125 -
126 - if (empty($bot_pinecone_config)) {
127 - // Fallback to default configuration
128 - return self::is_pinecone_enabled();
129 - }
130 -
131 - $enabled_check = !empty($bot_pinecone_config['use_pinecone']) && $bot_pinecone_config['use_pinecone'];
132 - $api_key_check = !empty($bot_pinecone_config['api_key']);
133 - $host_check = !empty($bot_pinecone_config['host']);
134 -
135 - return $enabled_check && $api_key_check && $host_check;
136 -}
137 -
138 -/**
139 - * Check if Pinecone is enabled and properly configured (original method for default bot)
140 - */
141 -private static function is_pinecone_enabled() {
142 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
143 -
144 - if (empty($pinecone_options)) {
145 - return false;
146 - }
147 -
148 - $enabled_check = !empty($pinecone_options['mxchat_use_pinecone']) && $pinecone_options['mxchat_use_pinecone'] !== '0';
149 - $api_key_check = !empty($pinecone_options['mxchat_pinecone_api_key']);
150 - $host_check = !empty($pinecone_options['mxchat_pinecone_host']);
151 -
152 - return $enabled_check && $api_key_check && $host_check;
153 -}
154 -
155 -/**
156 - * UPDATED: Store content in Pinecone only with bot support
157 - */
158 -private static function store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
159 - //error_log('[MXCHAT-PINECONE] ===== Using Pinecone-only storage for bot ' . $bot_id . ' =====');
160 -
161 - // Get bot-specific Pinecone configuration
162 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
163 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
164 - $api_key = $pinecone_options['mxchat_pinecone_api_key'];
165 - $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
166 - $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
167 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
168 - } else {
169 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
170 - if (empty($bot_pinecone_config)) {
171 - // Fallback to default configuration
172 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
173 - $api_key = $pinecone_options['mxchat_pinecone_api_key'];
174 - $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
175 - $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
176 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
177 - } else {
178 - $api_key = $bot_pinecone_config['api_key'];
179 - $environment = ''; // Not used in new Pinecone API
180 - $index_name = ''; // Not used in new Pinecone API
181 - $namespace = $bot_pinecone_config['namespace'] ?? '';
182 - }
183 - }
184 -
185 - $result = self::store_in_pinecone_main(
186 - $embedding_vector,
187 - $content,
188 - $source_url,
189 - $api_key,
190 - $environment,
191 - $index_name,
192 - $vector_id,
193 - $bot_id,
194 - $namespace,
195 - $content_type
196 - );
197 -
198 - if (is_wp_error($result)) {
199 - //error_log('[MXCHAT-PINECONE] Pinecone storage failed for bot ' . $bot_id . ': ' . $result->get_error_message());
200 - return $result;
201 - }
202 -
203 - //error_log('[MXCHAT-PINECONE] Pinecone storage completed successfully for bot ' . $bot_id);
204 - return true;
205 -}
206 -
207 -/**
208 - * Store content in WordPress database with progressive fallback
209 - * UPDATED 2.5.6: Now includes content_type parameter
210 - */
211 -private static function store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type = 'content') {
212 - global $wpdb;
213 -
214 - //error_log('[MXCHAT-DB] ===== Using WordPress-only storage =====');
215 -
216 - // Sanitize content_type
217 - $content_type = sanitize_key($content_type);
218 - if (empty($content_type)) {
219 - $content_type = 'content'; // Fallback for backwards compatibility
220 - }
221 -
222 - // ===== FIXED: Generate unique identifier for manual content =====
223 - $original_source_url = $source_url;
224 - // Check if this is truly manual content (no URL at all) vs a real URL that filter_var rejects
225 - // filter_var(FILTER_VALIDATE_URL) rejects valid URLs with encoded chars, non-ASCII, fragments, etc.
226 - // Use a looser check: if it starts with http(s):// or has a scheme, it's a URL
227 - $has_url_scheme = !empty($source_url) && preg_match('#^https?://#i', $source_url);
228 - // Treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
229 - $is_legacy_mxchat_url = $has_url_scheme && strpos($source_url, 'mxchat.ai') !== false;
230 - $is_manual_content = empty($source_url) || $source_url === '' || !$has_url_scheme || $is_legacy_mxchat_url;
231 -
232 - if ($is_manual_content) {
233 - // Generate unique identifier for manual content to prevent overwrites
234 - $source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
235 - //error_log('[MXCHAT-DB] Generated unique ID for manual content: ' . $source_url);
236 - }
237 -
238 - // Only check for duplicates if we have a valid source URL (not manual content)
239 - $existing_id = null;
240 - if (!$is_manual_content) {
241 - $existing_id = $wpdb->get_var(
242 - $wpdb->prepare(
243 - "SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
244 - $source_url
245 - )
246 - );
247 - //error_log('[MXCHAT-DB] Checked for existing URL, found ID: ' . ($existing_id ?: 'none'));
248 - } else {
249 - //error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)');
250 - }
251 - // ===== END FIX =====
252 -
253 - // Progressive fallback mechanism for problematic content
254 - $attempt = 1;
255 - $max_attempts = 3;
256 - $current_content = $safe_content;
257 - $result = false;
258 -
259 - while ($attempt <= $max_attempts && $result === false) {
260 - try {
261 - if ($existing_id) {
262 - //error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
263 -
264 - // Update the existing row - UPDATED 2.5.6: Added content_type
265 - $result = $wpdb->update(
266 - $table_name,
267 - array(
268 - 'url' => $source_url,
269 - 'article_content' => $current_content,
270 - 'embedding_vector' => $embedding_vector_serialized,
271 - 'source_url' => $source_url,
272 - 'content_type' => $content_type,
273 - 'timestamp' => current_time('mysql'),
274 - ),
275 - array('id' => $existing_id),
276 - array('%s','%s','%s','%s','%s','%s'),
277 - array('%d')
278 - );
279 - } else {
280 - //error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
281 - //error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
282 -
283 - // Insert a new row - UPDATED 2.5.6: Added content_type
284 - $result = $wpdb->insert(
285 - $table_name,
286 - array(
287 - 'url' => $source_url, // Now unique for manual content
288 - 'article_content' => $current_content,
289 - 'embedding_vector' => $embedding_vector_serialized,
290 - 'source_url' => $source_url, // Now unique for manual content
291 - 'content_type' => $content_type,
292 - 'timestamp' => current_time('mysql'),
293 - ),
294 - array('%s','%s','%s','%s','%s','%s')
295 - );
296 - }
297 -
298 - if ($result === false) {
299 - //error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . ')');
300 - //error_log('[MXCHAT-DB] MySQL Error: ' . $wpdb->last_error);
301 - //error_log('[MXCHAT-DB] MySQL Error Number: ' . $wpdb->last_errno);
302 - //error_log('[MXCHAT-DB] Last Query: ' . substr($wpdb->last_query, 0, 500));
303 - //error_log('[MXCHAT-DB] Content length: ' . strlen($current_content) . ' bytes');
304 - //error_log('[MXCHAT-DB] Embedding vector length: ' . strlen($embedding_vector_serialized) . ' bytes');
305 -
306 - // Progressively apply more aggressive sanitization on failure
307 - if ($attempt === 1) {
308 - // First fallback: Use a more aggressive character filter and shorten
309 - $current_content = preg_replace('/[^\p{L}\p{N}\s.,;:!?()-]/u', '', $current_content);
310 - $current_content = substr($current_content, 0, 50000);
311 - } else if ($attempt === 2) {
312 - // Second fallback: Keep only alphanumeric and basic punctuation, shorten further
313 - $current_content = preg_replace('/[^a-zA-Z0-9\s.,;:!?()-]/u', '', $current_content);
314 - $current_content = substr($current_content, 0, 30000);
315 - }
316 -
317 - $attempt++;
318 - }
319 - } catch (Exception $e) {
320 - //error_log('[MXCHAT-DB] Exception during database operation: ' . $e->getMessage());
321 - $attempt++;
322 - }
323 - }
324 -
325 -if ($result === false) {
326 - //error_log('[MXCHAT-DB] All database operation attempts failed');
327 - //error_log('[MXCHAT-DB] Final MySQL Error: ' . $wpdb->last_error);
328 -
329 - $detailed_error = sprintf(
330 - 'Failed to store content in WordPress database after %d attempts. MySQL Error: %s (Error #%d). Content size: %d bytes, Embedding size: %d bytes',
331 - $max_attempts,
332 - $wpdb->last_error,
333 - $wpdb->last_errno,
334 - strlen($current_content),
335 - strlen($embedding_vector_serialized)
336 - );
337 -
338 - return new WP_Error('database_failed', $detailed_error);
339 -}
340 -
341 - //error_log('[MXCHAT-DB] WordPress database operation completed successfully (Attempt ' . ($attempt - 1) . ')');
342 - return true;
343 -}
344 -
345 -/**
346 - * UPDATED: Store content in Pinecone database with bot support
347 - * UPDATED 2.5.6: Now accepts content_type parameter
348 - */
349 -private static function store_in_pinecone_main($embedding_vector, $content, $url, $api_key, $environment, $index_name, $vector_id = null, $bot_id = 'default', $namespace = '', $content_type = 'content') {
350 - //error_log('[MXCHAT-PINECONE-MAIN] ===== Starting Pinecone storage for bot ' . $bot_id . ' =====');
351 -
352 - // ===== UPDATED: Handle manual content with unique vector IDs =====
353 - if ($vector_id) {
354 - // Use provided vector ID
355 - //error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
356 - } elseif (!empty($url) && preg_match('#^https?://#i', $url)) {
357 - // For URLs, use URL-based ID (existing behavior)
358 - $vector_id = md5($url);
359 - //error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
360 - } else {
361 - // For manual content (empty/no URL scheme), generate unique ID
362 - $vector_id = 'manual_' . time() . '_' . substr(md5($content . microtime(true)), 0, 8);
363 - //error_log('[MXCHAT-PINECONE-MAIN] Generated unique vector ID for manual content: ' . $vector_id);
364 - }
365 - // ===== END UPDATE =====
366 -
367 - // Get host from bot-specific config or fallback to default
368 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
369 - $options = get_option('mxchat_pinecone_addon_options');
370 - $host = $options['mxchat_pinecone_host'] ?? '';
371 - } else {
372 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
373 - if (!empty($bot_pinecone_config)) {
374 - $host = $bot_pinecone_config['host'] ?? '';
375 - } else {
376 - $options = get_option('mxchat_pinecone_addon_options');
377 - $host = $options['mxchat_pinecone_host'] ?? '';
378 - }
379 - }
380 -
381 - //error_log('[MXCHAT-PINECONE-MAIN] Host: ' . $host);
382 - //error_log('[MXCHAT-PINECONE-MAIN] API key length: ' . strlen($api_key));
383 - //error_log('[MXCHAT-PINECONE-MAIN] Bot ID: ' . $bot_id);
384 - //error_log('[MXCHAT-PINECONE-MAIN] Namespace: ' . $namespace);
385 -
386 - if (empty($host)) {
387 - //error_log('[MXCHAT-PINECONE-MAIN] ERROR: Host is empty');
388 - return new WP_Error('pinecone_config', 'Pinecone host is not configured. Please set the host in your bot settings.');
389 - }
390 -
391 - // ===== UPDATED 2.5.6: Use passed content_type or determine from URL if not provided =====
392 - // Sanitize content_type
393 - $content_type = sanitize_key($content_type);
394 - if (empty($content_type)) {
395 - // Fallback to old detection logic for backwards compatibility
396 - $is_product = false;
397 - $content_type = 'manual'; // Default for manual content
398 -
399 - if (!empty($url) && preg_match('#^https?://#i', $url)) {
400 - $is_product = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
401 - $content_type = $is_product ? 'product' : 'content';
402 - }
403 - }
404 -
405 - //error_log('[MXCHAT-PINECONE-MAIN] Content type: ' . $content_type);
406 - // ===== END UPDATE =====
407 -
408 - $api_endpoint = "https://{$host}/vectors/upsert";
409 - //error_log('[MXCHAT-PINECONE-MAIN] API endpoint: ' . $api_endpoint);
410 -
411 - // UPDATED 2.5.6: Use provided content_type in metadata
412 - $metadata = array(
413 - 'text' => $content,
414 - 'source_url' => $url, // Can be empty for manual content
415 - 'type' => $content_type, // Now supports: post, page, pdf, url, manual, product, etc.
416 - 'last_updated' => time(),
417 - 'created_at' => time(), // Add creation timestamp
418 - 'bot_id' => $bot_id, // Add bot identification
419 - );
420 -
421 - $vector_data = array(
422 - 'id' => $vector_id,
423 - 'values' => $embedding_vector,
424 - 'metadata' => $metadata
425 - );
426 -
427 - $request_body = array(
428 - 'vectors' => array($vector_data)
429 - );
430 -
431 - // Add namespace if specified for multi-bot separation
432 - if (!empty($namespace)) {
433 - $request_body['namespace'] = $namespace;
434 - //error_log('[MXCHAT-PINECONE-MAIN] Using namespace: ' . $namespace);
435 - }
436 -
437 - //error_log('[MXCHAT-PINECONE-MAIN] Request body prepared (embedding dimensions: ' . count($embedding_vector) . ')');
438 -
439 - $response = wp_remote_post($api_endpoint, array(
440 - 'headers' => array(
441 - 'Api-Key' => $api_key,
442 - 'accept' => 'application/json',
443 - 'content-type' => 'application/json'
444 - ),
445 - 'body' => wp_json_encode($request_body),
446 - 'timeout' => 30,
447 - 'data_format' => 'body'
448 - ));
449 -
450 - if (is_wp_error($response)) {
451 - //error_log('[MXCHAT-PINECONE-MAIN] WordPress request error: ' . $response->get_error_message());
452 - return new WP_Error('pinecone_request', $response->get_error_message());
453 - }
454 -
455 - $response_code = wp_remote_retrieve_response_code($response);
456 - //error_log('[MXCHAT-PINECONE-MAIN] Response code: ' . $response_code);
457 -
458 - if ($response_code !== 200) {
459 - $body = wp_remote_retrieve_body($response);
460 - //error_log('[MXCHAT-PINECONE-MAIN] API error - Response body: ' . $body);
461 - return new WP_Error('pinecone_api', sprintf(
462 - 'Pinecone API error (HTTP %d): %s',
463 - $response_code,
464 - $body
465 - ));
466 - }
467 -
468 - $response_body = wp_remote_retrieve_body($response);
469 - //error_log('[MXCHAT-PINECONE-MAIN] Success response: ' . $response_body);
470 - //error_log('[MXCHAT-PINECONE-MAIN] Successfully stored in Pinecone for bot ' . $bot_id);
471 - //error_log('[MXCHAT-PINECONE-MAIN] ===== Pinecone storage complete =====');
472 -
473 - return true;
474 -}
475 -
476 -/**
477 - * UPDATED: Generate an embedding for the given text using bot-specific configuration.
478 - *
479 - * @param string $text The text to be embedded.
480 - * @param string $api_key The API key used for generating embeddings.
481 - * @param string $bot_id The bot ID for multi-bot support
482 - * @return array|null The embedding vector or null on failure.
483 - */
484 -private static function generate_embedding($text, $api_key, $bot_id = 'default') {
485 - // Get bot-specific options
486 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
487 - $options = get_option('mxchat_options');
488 - } else {
489 - $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
490 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
491 - }
492 -
493 - // Opt-in: when the custom provider is selected for embeddings, route the KB
494 - // INDEX side through the same custom endpoint the query side uses, so stored
495 - // vectors and query vectors come from the same model. Default-off behavior
496 - // below is untouched.
497 - if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
498 - $custom = self::generate_embedding_custom($text, $options);
499 - return is_array($custom) ? $custom : null;
500 - }
501 -
502 - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
503 -
504 - // Determine endpoint and API key based on model
505 - if (strpos($selected_model, 'voyage') === 0) {
506 - $endpoint = 'https://api.voyageai.com/v1/embeddings';
507 - $api_key = $options['voyage_api_key'] ?? '';
508 - } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
509 - $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
510 - $api_key = $options['gemini_api_key'] ?? '';
511 - } else {
512 - $endpoint = 'https://api.openai.com/v1/embeddings';
513 - // Use the bot-specific API key or fallback to passed API key
514 - $api_key = $options['api_key'] ?? $api_key;
515 - }
516 -
517 - // Prepare request body based on provider
518 - if (strpos($selected_model, 'gemini-embedding') === 0) {
519 - // Gemini API format
520 - $request_body = [
521 - 'model' => 'models/' . $selected_model,
522 - 'content' => [
523 - 'parts' => [
524 - ['text' => $text]
525 - ]
526 - ],
527 - 'outputDimensionality' => 1536
528 - ];
529 -
530 - // Prepare headers for Gemini (API key as query parameter)
531 - $endpoint .= '?key=' . $api_key;
532 - $headers = [
533 - 'Content-Type' => 'application/json'
534 - ];
535 - } else {
536 - // OpenAI/Voyage API format
537 - $request_body = [
538 - 'input' => $text,
539 - 'model' => $selected_model
540 - ];
541 -
542 - // Add output_dimension for voyage-3-large
543 - if ($selected_model === 'voyage-3-large') {
544 - $request_body['output_dimension'] = 2048;
545 - }
546 -
547 - // Prepare headers for OpenAI/Voyage
548 - $headers = [
549 - 'Content-Type' => 'application/json',
550 - 'Authorization' => 'Bearer ' . $api_key
551 - ];
552 - }
553 -
554 - $args = [
555 - 'body' => wp_json_encode($request_body),
556 - 'headers' => $headers,
557 - 'timeout' => 60,
558 - 'redirection' => 5,
559 - 'blocking' => true,
560 - 'httpversion' => '1.0',
561 - 'sslverify' => true,
562 - ];
563 -
564 - $response = wp_remote_post($endpoint, $args);
565 -
566 - if (is_wp_error($response)) {
567 - //error_log('Error generating embedding for bot ' . $bot_id . ': ' . $response->get_error_message());
568 - return null;
569 - }
570 -
571 - $response_body = json_decode(wp_remote_retrieve_body($response), true);
572 -
573 - // Handle different response formats based on provider
574 - if (strpos($selected_model, 'gemini-embedding') === 0) {
575 - // Gemini API response format
576 - if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
577 - self::stamp_active_embedding_model($selected_model);
578 - return $response_body['embedding']['values'];
579 - } else {
580 - //error_log('Invalid response received from Gemini embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
581 - return null;
582 - }
583 - } else {
584 - // OpenAI/Voyage API response format
585 - if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
586 - self::stamp_active_embedding_model($selected_model);
587 - return $response_body['data'][0]['embedding'];
588 - } else {
589 - //error_log('Invalid response received from embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
590 - return null;
591 - }
592 - }
593 -}
594 -
595 -/**
596 - * Generate an embedding via a Custom (OpenAI-compatible) provider's /embeddings route.
597 - * Shared by every embedding entry point so the KNOWLEDGE-BASE INDEX side and the
598 - * QUERY side route through the same model when the opt-in
599 - * 'custom_provider_for_embeddings' setting is on. Mirrors the query-path logic in
600 - * MxChat_Integrator::mxchat_generate_embedding_custom() but takes an explicit
601 - * $options array so it is callable statically from utils + knowledge-manager.
602 - *
603 - * Returns a numeric array (the embedding vector) on success, or a human-readable
604 - * error string on failure (so callers expecting a string error, like the
605 - * knowledge-manager, can surface it directly; callers expecting array|null wrap it).
606 - *
607 - * @param string $text Text to embed.
608 - * @param array $options The resolved mxchat options (must contain the custom_provider_* keys).
609 - * @return array|string Embedding vector on success; error string on failure.
610 - */
611 -public static function generate_embedding_custom($text, $options) {
612 - if (empty($text)) {
613 - return 'No text provided for embedding generation';
614 - }
615 -
616 - $base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
617 - if (empty($base_url)) {
618 - return 'Custom provider Base URL is not configured.';
619 - }
620 -
621 - $api_key = isset($options['custom_provider_api_key']) ? trim((string) $options['custom_provider_api_key']) : '';
622 - $auth_scheme = isset($options['custom_provider_auth_scheme']) ? $options['custom_provider_auth_scheme'] : 'bearer';
623 - $api_version = isset($options['custom_provider_api_version']) ? trim((string) $options['custom_provider_api_version']) : '';
624 -
625 - // Embedding model: prefer the dedicated custom_provider_embedding_model, fall back to the chat model.
626 - $model = (isset($options['custom_provider_embedding_model']) && trim((string) $options['custom_provider_embedding_model']) !== '')
627 - ? trim((string) $options['custom_provider_embedding_model'])
628 - : ((isset($options['custom_provider_model']) && trim((string) $options['custom_provider_model']) !== '') ? trim((string) $options['custom_provider_model']) : 'default');
629 -
630 - $embed_url = $base_url . '/embeddings';
631 - if (!empty($api_version)) {
632 - $embed_url .= (strpos($embed_url, '?') === false ? '?' : '&') . 'api-version=' . rawurlencode($api_version);
633 - }
634 -
635 - $headers = ['Content-Type' => 'application/json'];
636 - if (!empty($api_key)) {
637 - if ($auth_scheme === 'api-key') {
638 - $headers['api-key'] = $api_key;
639 - } else {
640 - $headers['Authorization'] = 'Bearer ' . $api_key;
641 - }
642 - }
643 -
644 - $response = wp_remote_post($embed_url, [
645 - 'headers' => $headers,
646 - 'body' => wp_json_encode(['input' => $text, 'model' => $model]),
647 - 'timeout' => 60,
648 - ]);
649 - if (is_wp_error($response)) {
650 - return 'Connection error when generating embeddings (custom provider): ' . $response->get_error_message();
651 - }
652 -
653 - $status = wp_remote_retrieve_response_code($response);
654 - $body = json_decode(wp_remote_retrieve_body($response), true);
655 - if ($status !== 200) {
656 - $msg = isset($body['error']['message']) ? $body['error']['message'] : 'HTTP ' . $status;
657 - return 'Custom embedding endpoint error: ' . $msg;
658 - }
659 - if (isset($body['data'][0]['embedding']) && is_array($body['data'][0]['embedding'])) {
660 - // Stamp the custom model identity so the active-embedding-model mismatch
661 - // warning reflects the real (custom) model rather than the built-in setting.
662 - self::stamp_active_embedding_model('custom:' . $model);
663 - return $body['data'][0]['embedding'];
664 - }
665 - return 'Invalid embedding response from custom provider.';
666 -}
667 -
668 -/**
669 - * Submit content as multiple chunks
670 - *
671 - * Splits large content into chunks, generates embeddings for each,
672 - * and stores them with chunk metadata for later reassembly.
673 - *
674 - * @param string $content The content to chunk and store
675 - * @param string $source_url The source URL
676 - * @param string $api_key The API key for embeddings
677 - * @param string $bot_id The bot ID
678 - * @param string $content_type The content type
679 - * @param MxChat_Chunker $chunker The chunker instance
680 - * @return bool|WP_Error True on success, WP_Error on failure
681 - */
682 -private static function submit_chunked_content($content, $source_url, $api_key, $bot_id, $content_type, $chunker) {
683 - global $wpdb;
684 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
685 -
686 - //error_log('[MXCHAT-CHUNK-DEBUG] Starting chunked submission for: ' . $source_url);
687 - //error_log('[MXCHAT-CHUNK-DEBUG] Content length: ' . strlen($content) . ' chars');
688 -
689 - // First, delete any existing chunks for this URL (clean slate)
690 - $delete_result = self::delete_chunks_for_url($source_url, $bot_id);
691 - if (is_wp_error($delete_result)) {
692 - //error_log('[MXCHAT-CHUNK-DEBUG] Warning: Failed to delete existing chunks: ' . $delete_result->get_error_message());
693 - // Continue anyway - we'll overwrite with upsert
694 - }
695 -
696 - // Split content into chunks
697 - $chunks = $chunker->chunk_text($content);
698 - $total_chunks = count($chunks);
699 -
700 - //error_log('[MXCHAT-CHUNK-DEBUG] Created ' . $total_chunks . ' chunks');
701 - foreach ($chunks as $i => $chunk) {
702 - //error_log('[MXCHAT-CHUNK-DEBUG] Chunk ' . $i . ' length: ' . strlen($chunk) . ' chars, preview: ' . substr($chunk, 0, 100));
703 - }
704 -
705 - //error_log('[MXCHAT-CHUNK] Split content into ' . $total_chunks . ' chunks');
706 -
707 - if ($total_chunks === 0) {
708 - return new WP_Error('chunking_failed', 'Content could not be split into chunks');
709 - }
710 -
711 - $errors = array();
712 - $is_pinecone = self::is_pinecone_enabled_for_bot($bot_id);
713 -
714 - foreach ($chunks as $index => $chunk_text) {
715 - // Generate chunk metadata
716 - $chunk_metadata = MxChat_Chunker::create_chunk_metadata($index, $total_chunks, $source_url);
717 -
718 - // AI-Engine-style aliases so external consumers (Pinecone/Qdrant/Chroma) can rely on
719 - // a stable shorthand ('source'/'part_index'/'part_total') without parsing our internal names.
720 - $chunk_metadata['source'] = $source_url;
721 - $chunk_metadata['part_index'] = (int) $index;
722 - $chunk_metadata['part_total'] = (int) $total_chunks;
723 -
724 - /**
725 - * Filter the per-chunk metadata blob before it's written to the KB store.
726 - *
727 - * @param array $chunk_metadata Metadata array (source, part_index, part_total, chunk_index, total_chunks, source_url, parent_url_hash, document_type, ...).
728 - * @param string $chunk_text The chunk text being stored.
729 - * @param array $context ['bot_id' => string, 'content_type' => string, 'source_url' => string, 'part_index' => int, 'part_total' => int]
730 - * @return array Updated metadata array.
731 - */
732 - $chunk_metadata = apply_filters(
733 - 'mxchat_embedding_chunk_metadata',
734 - $chunk_metadata,
735 - $chunk_text,
736 - array(
737 - 'bot_id' => $bot_id,
738 - 'content_type' => $content_type,
739 - 'source_url' => $source_url,
740 - 'part_index' => (int) $index,
741 - 'part_total' => (int) $total_chunks,
742 - )
743 - );
744 -
745 - $chunk_vector_id = MxChat_Chunker::generate_chunk_vector_id($source_url, $index);
746 -
747 - //error_log('[MXCHAT-CHUNK] Processing chunk ' . ($index + 1) . '/' . $total_chunks . ' (ID: ' . $chunk_vector_id . ')');
748 -
749 - // Generate embedding for this chunk
750 - $embedding_vector = self::generate_embedding($chunk_text, $api_key, $bot_id);
751 -
752 - if (!is_array($embedding_vector)) {
753 - $errors[] = new WP_Error('embedding_failed', 'Failed to generate embedding for chunk ' . $index);
754 - //error_log('[MXCHAT-CHUNK] Failed to generate embedding for chunk ' . $index);
755 - continue;
756 - }
757 -
758 - if ($is_pinecone) {
759 - // Store in Pinecone with chunk metadata
760 - $result = self::store_chunk_in_pinecone(
761 - $embedding_vector,
762 - $chunk_text,
763 - $source_url,
764 - $chunk_vector_id,
765 - $bot_id,
766 - $content_type,
767 - $chunk_metadata
768 - );
769 - } else {
770 - // Store in WordPress DB with chunk metadata
771 - $content_with_metadata = MxChat_Chunker::format_chunk_for_storage($chunk_text, $chunk_metadata);
772 - $embedding_vector_serialized = maybe_serialize($embedding_vector);
773 -
774 - $result = self::store_chunk_in_wordpress_db(
775 - $content_with_metadata,
776 - $source_url,
777 - $embedding_vector_serialized,
778 - $table_name,
779 - $content_type,
780 - $chunk_metadata
781 - );
782 - }
783 -
784 - if (is_wp_error($result)) {
785 - $errors[] = $result;
786 - //error_log('[MXCHAT-CHUNK] Failed to store chunk ' . $index . ': ' . $result->get_error_message());
787 - }
788 - }
789 -
790 - if (count($errors) === $total_chunks) {
791 - return new WP_Error('chunking_failed', 'Failed to store any chunks');
792 - }
793 -
794 - if (!empty($errors)) {
795 - //error_log('[MXCHAT-CHUNK] Completed with ' . count($errors) . ' errors out of ' . $total_chunks . ' chunks');
796 - return new WP_Error('chunking_partial_failure',
797 - sprintf('Failed to store %d of %d chunks', count($errors), $total_chunks));
798 - }
799 -
800 - //error_log('[MXCHAT-CHUNK] Successfully stored all ' . $total_chunks . ' chunks');
801 - return true;
802 -}
803 -
804 -/**
805 - * Store a single chunk in Pinecone with chunk-specific metadata
806 - */
807 -private static function store_chunk_in_pinecone($embedding_vector, $chunk_text, $source_url, $vector_id, $bot_id, $content_type, $chunk_metadata) {
808 - // Get Pinecone configuration
809 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
810 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
811 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
812 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
813 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
814 - } else {
815 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
816 - if (empty($bot_pinecone_config)) {
817 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
818 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
819 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
820 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
821 - } else {
822 - $api_key = $bot_pinecone_config['api_key'] ?? '';
823 - $host = $bot_pinecone_config['host'] ?? '';
824 - $namespace = $bot_pinecone_config['namespace'] ?? '';
825 - }
826 - }
827 -
828 - if (empty($host) || empty($api_key)) {
829 - return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
830 - }
831 -
832 - $api_endpoint = "https://{$host}/vectors/upsert";
833 -
834 - // Build metadata with chunk information
835 - $metadata = array(
836 - 'text' => $chunk_text,
837 - 'source_url' => $source_url,
838 - 'type' => $content_type,
839 - 'is_chunked' => true,
840 - 'chunk_index' => $chunk_metadata['chunk_index'],
841 - 'total_chunks' => $chunk_metadata['total_chunks'],
842 - 'parent_url_hash' => $chunk_metadata['parent_url_hash'],
843 - 'last_updated' => time(),
844 - 'created_at' => time(),
845 - 'bot_id' => $bot_id,
846 - );
847 -
848 - $vector_data = array(
849 - 'id' => $vector_id,
850 - 'values' => $embedding_vector,
851 - 'metadata' => $metadata
852 - );
853 -
854 - $request_body = array(
855 - 'vectors' => array($vector_data)
856 - );
857 -
858 - if (!empty($namespace)) {
859 - $request_body['namespace'] = $namespace;
860 - }
861 -
862 - $response = wp_remote_post($api_endpoint, array(
863 - 'headers' => array(
864 - 'Api-Key' => $api_key,
865 - 'accept' => 'application/json',
866 - 'content-type' => 'application/json'
867 - ),
868 - 'body' => wp_json_encode($request_body),
869 - 'timeout' => 30
870 - ));
871 -
872 - if (is_wp_error($response)) {
873 - return $response;
874 - }
875 -
876 - $response_code = wp_remote_retrieve_response_code($response);
877 - if ($response_code !== 200) {
878 - return new WP_Error('pinecone_api', 'Pinecone API error: HTTP ' . $response_code);
879 - }
880 -
881 - return true;
882 -}
883 -
884 -/**
885 - * Store a single chunk in WordPress database
886 - */
887 -private static function store_chunk_in_wordpress_db($content_with_metadata, $source_url, $embedding_vector_serialized, $table_name, $content_type, $chunk_metadata) {
888 - global $wpdb;
889 -
890 - // For chunks, we always insert new rows (no duplicate checking)
891 - // The URL includes chunk info in the metadata, but source_url stays the same for grouping
892 - $result = $wpdb->insert(
893 - $table_name,
894 - array(
895 - 'url' => $source_url,
896 - 'article_content' => $content_with_metadata,
897 - 'embedding_vector' => $embedding_vector_serialized,
898 - 'source_url' => $source_url,
899 - 'content_type' => $content_type,
900 - 'timestamp' => current_time('mysql')
901 - ),
902 - array('%s', '%s', '%s', '%s', '%s', '%s')
903 - );
904 -
905 - if ($result === false) {
906 - return new WP_Error('database_failed', 'Failed to insert chunk: ' . $wpdb->last_error);
907 - }
908 -
909 - return true;
910 -}
911 -
912 -/**
913 - * Delete all chunks for a given URL
914 - *
915 - * @param string $source_url The source URL
916 - * @param string $bot_id The bot ID
917 - * @return bool|WP_Error True on success, WP_Error on failure
918 - */
919 -public static function delete_chunks_for_url($source_url, $bot_id = 'default') {
920 - //error_log('[MXCHAT-CHUNK-DELETE] Deleting chunks for URL: ' . $source_url);
921 -
922 - if (self::is_pinecone_enabled_for_bot($bot_id)) {
923 - return self::delete_pinecone_chunks_by_url($source_url, $bot_id);
924 - } else {
925 - return self::delete_wordpress_chunks_by_url($source_url);
926 - }
927 -}
928 -
929 -/**
930 - * Delete all chunks for a URL from Pinecone
931 - */
932 -private static function delete_pinecone_chunks_by_url($source_url, $bot_id) {
933 - // Get Pinecone configuration
934 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
935 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
936 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
937 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
938 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
939 - } else {
940 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
941 - if (empty($bot_pinecone_config)) {
942 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
943 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
944 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
945 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
946 - } else {
947 - $api_key = $bot_pinecone_config['api_key'] ?? '';
948 - $host = $bot_pinecone_config['host'] ?? '';
949 - $namespace = $bot_pinecone_config['namespace'] ?? '';
950 - }
951 - }
952 -
953 - if (empty($host) || empty($api_key)) {
954 - return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
955 - }
956 -
957 - $base_vector_id = md5($source_url);
958 - $vectors_to_delete = array();
959 -
960 - // Add the original single-vector ID (for non-chunked content)
961 - $vectors_to_delete[] = $base_vector_id;
962 -
963 - // Pinecone /vectors/list is a GET endpoint with query-string parameters; a POST here returns a
964 - // non-200 silently and we end up only deleting the base vector, leaving chunks orphaned.
965 - $query_params = array(
966 - 'prefix' => $base_vector_id . '_chunk_',
967 - 'limit' => 100,
968 - );
969 - if (!empty($namespace)) {
970 - $query_params['namespace'] = $namespace;
971 - }
972 -
973 - $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
974 -
975 - // Paginate in case a URL has more than 100 chunks.
976 - do {
977 - $list_response = wp_remote_get($list_url, array(
978 - 'headers' => array(
979 - 'Api-Key' => $api_key,
980 - 'accept' => 'application/json',
981 - ),
982 - 'timeout' => 30,
983 - ));
984 -
985 - if (is_wp_error($list_response) || wp_remote_retrieve_response_code($list_response) !== 200) {
986 - break;
987 - }
988 -
989 - $list_data = json_decode(wp_remote_retrieve_body($list_response), true);
990 - if (!empty($list_data['vectors'])) {
991 - foreach ($list_data['vectors'] as $vector) {
992 - if (isset($vector['id'])) {
993 - $vectors_to_delete[] = $vector['id'];
994 - }
995 - }
996 - }
997 -
998 - $next_token = $list_data['pagination']['next'] ?? '';
999 - if (empty($next_token)) {
1000 - break;
1001 - }
1002 -
1003 - $query_params['paginationToken'] = $next_token;
1004 - $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
1005 - } while (true);
1006 -
1007 - if (empty($vectors_to_delete)) {
1008 - //error_log('[MXCHAT-CHUNK-DELETE] No vectors found to delete');
1009 - return true;
1010 - }
1011 -
1012 - //error_log('[MXCHAT-CHUNK-DELETE] Deleting ' . count($vectors_to_delete) . ' vectors from Pinecone');
1013 -
1014 - // Delete vectors
1015 - $delete_url = "https://{$host}/vectors/delete";
1016 -
1017 - $delete_body = array(
1018 - 'ids' => $vectors_to_delete
1019 - );
1020 -
1021 - if (!empty($namespace)) {
1022 - $delete_body['namespace'] = $namespace;
1023 - }
1024 -
1025 - $delete_response = wp_remote_post($delete_url, array(
1026 - 'headers' => array(
1027 - 'Api-Key' => $api_key,
1028 - 'accept' => 'application/json',
1029 - 'content-type' => 'application/json'
1030 - ),
1031 - 'body' => wp_json_encode($delete_body),
1032 - 'timeout' => 30
1033 - ));
1034 -
1035 - if (is_wp_error($delete_response)) {
1036 - return $delete_response;
1037 - }
1038 -
1039 - $response_code = wp_remote_retrieve_response_code($delete_response);
1040 - if ($response_code !== 200) {
1041 - return new WP_Error('pinecone_delete', 'Failed to delete vectors: HTTP ' . $response_code);
1042 - }
1043 -
1044 - return true;
1045 -}
1046 -
1047 -/**
1048 - * Delete all chunks for a URL from WordPress database
1049 - */
1050 -private static function delete_wordpress_chunks_by_url($source_url) {
1051 - global $wpdb;
1052 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1053 -
1054 - // Delete all rows with this source_url (handles both chunked and non-chunked)
1055 - $result = $wpdb->delete(
1056 - $table_name,
1057 - array('source_url' => $source_url),
1058 - array('%s')
1059 - );
1060 -
1061 - if ($result === false) {
1062 - return new WP_Error('database_delete', 'Failed to delete chunks: ' . $wpdb->last_error);
1063 - }
1064 -
1065 - //error_log('[MXCHAT-CHUNK-DELETE] Deleted ' . $result . ' rows from WordPress DB');
1066 - return true;
1067 -}
1 +<?php
2 +if (!defined('ABSPATH')) {
3 + exit; // Exit if accessed directly
4 +}
5 +
6 +class MxChat_Utils {
7 +
8 +/**
9 + * UPDATED: Submit or update content (and its embedding) in the database.
10 + * Stores in Pinecone if enabled, otherwise stores in WordPress DB.
11 + *
12 + * @param string $content The content to be embedded.
13 + * @param string $source_url The source URL of the content.
14 + * @param string $api_key The API key used for generating embeddings.
15 + * @param string $vector_id Optional vector ID for Pinecone (if not provided, will use md5 of URL)
16 + * @param string $bot_id The bot ID for multi-bot support
17 + * @return bool|WP_Error True on success, WP_Error on failure
18 + */
19 +public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default') {
20 + global $wpdb;
21 + $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
22 +
23 + //error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ')');
24 + //error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
25 +
26 + // Sanitize the source URL
27 + $source_url = esc_url_raw($source_url);
28 +
29 + // Just ensure UTF-8 validity without aggressive escaping
30 + $safe_content = wp_check_invalid_utf8($content);
31 + // Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D)
32 + $safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
33 +
34 + // UPDATED: Generate the embedding using bot-specific configuration
35 + $embedding_vector = self::generate_embedding($content, $api_key, $bot_id);
36 +
37 + if (!is_array($embedding_vector)) {
38 + //error_log('[MXCHAT-DB] Error: Embedding generation failed');
39 + return new WP_Error('embedding_failed', 'Failed to generate embedding for content');
40 + }
41 +
42 + //error_log('[MXCHAT-DB] Embedding generated successfully');
43 +
44 + // UPDATED: Check if Pinecone is enabled for this specific bot
45 + if (self::is_pinecone_enabled_for_bot($bot_id)) {
46 + //error_log('[MXCHAT-DB] Pinecone is enabled for bot ' . $bot_id . ' - using Pinecone storage');
47 + // Store in Pinecone only
48 + return self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id);
49 + } else {
50 + //error_log('[MXCHAT-DB] Pinecone not enabled for bot ' . $bot_id . ' - using WordPress storage');
51 + // Store in WordPress database only
52 + $embedding_vector_serialized = maybe_serialize($embedding_vector);
53 + return self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name);
54 + }
55 +}
56 +
57 +/**
58 + * UPDATED: Check if Pinecone is enabled and properly configured for a specific bot
59 + */
60 +private static function is_pinecone_enabled_for_bot($bot_id = 'default') {
61 + // For default bot or when multi-bot is not active, use original method
62 + if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
63 + return self::is_pinecone_enabled();
64 + }
65 +
66 + // Get bot-specific Pinecone configuration
67 + $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
68 +
69 + if (empty($bot_pinecone_config)) {
70 + // Fallback to default configuration
71 + return self::is_pinecone_enabled();
72 + }
73 +
74 + $enabled_check = !empty($bot_pinecone_config['use_pinecone']) && $bot_pinecone_config['use_pinecone'];
75 + $api_key_check = !empty($bot_pinecone_config['api_key']);
76 + $host_check = !empty($bot_pinecone_config['host']);
77 +
78 + return $enabled_check && $api_key_check && $host_check;
79 +}
80 +
81 +/**
82 + * Check if Pinecone is enabled and properly configured (original method for default bot)
83 + */
84 +private static function is_pinecone_enabled() {
85 + $pinecone_options = get_option('mxchat_pinecone_addon_options');
86 +
87 + if (empty($pinecone_options)) {
88 + return false;
89 + }
90 +
91 + $enabled_check = !empty($pinecone_options['mxchat_use_pinecone']) && $pinecone_options['mxchat_use_pinecone'] !== '0';
92 + $api_key_check = !empty($pinecone_options['mxchat_pinecone_api_key']);
93 + $host_check = !empty($pinecone_options['mxchat_pinecone_host']);
94 +
95 + return $enabled_check && $api_key_check && $host_check;
96 +}
97 +
98 +/**
99 + * UPDATED: Store content in Pinecone only with bot support
100 + */
101 +private static function store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id = null, $bot_id = 'default') {
102 + //error_log('[MXCHAT-PINECONE] ===== Using Pinecone-only storage for bot ' . $bot_id . ' =====');
103 +
104 + // Get bot-specific Pinecone configuration
105 + if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
106 + $pinecone_options = get_option('mxchat_pinecone_addon_options');
107 + $api_key = $pinecone_options['mxchat_pinecone_api_key'];
108 + $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
109 + $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
110 + $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
111 + } else {
112 + $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
113 + if (empty($bot_pinecone_config)) {
114 + // Fallback to default configuration
115 + $pinecone_options = get_option('mxchat_pinecone_addon_options');
116 + $api_key = $pinecone_options['mxchat_pinecone_api_key'];
117 + $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
118 + $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
119 + $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
120 + } else {
121 + $api_key = $bot_pinecone_config['api_key'];
122 + $environment = ''; // Not used in new Pinecone API
123 + $index_name = ''; // Not used in new Pinecone API
124 + $namespace = $bot_pinecone_config['namespace'] ?? '';
125 + }
126 + }
127 +
128 + $result = self::store_in_pinecone_main(
129 + $embedding_vector,
130 + $content,
131 + $source_url,
132 + $api_key,
133 + $environment,
134 + $index_name,
135 + $vector_id,
136 + $bot_id,
137 + $namespace
138 + );
139 +
140 + if (is_wp_error($result)) {
141 + //error_log('[MXCHAT-PINECONE] Pinecone storage failed for bot ' . $bot_id . ': ' . $result->get_error_message());
142 + return $result;
143 + }
144 +
145 + //error_log('[MXCHAT-PINECONE] Pinecone storage completed successfully for bot ' . $bot_id);
146 + return true;
147 +}
148 +
149 +/**
150 + * Store content in WordPress database with progressive fallback (unchanged)
151 + */
152 +private static function store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name) {
153 + global $wpdb;
154 +
155 + //error_log('[MXCHAT-DB] ===== Using WordPress-only storage =====');
156 +
157 + // ===== FIXED: Generate unique identifier for manual content =====
158 + $original_source_url = $source_url;
159 + $is_manual_content = empty($source_url) || $source_url === '' || !filter_var($source_url, FILTER_VALIDATE_URL);
160 +
161 + if ($is_manual_content) {
162 + // Generate unique identifier for manual content to prevent overwrites
163 + $source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
164 + //error_log('[MXCHAT-DB] Generated unique ID for manual content: ' . $source_url);
165 + }
166 +
167 + // Only check for duplicates if we have a valid source URL (not manual content)
168 + $existing_id = null;
169 + if (!$is_manual_content) {
170 + $existing_id = $wpdb->get_var(
171 + $wpdb->prepare(
172 + "SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
173 + $source_url
174 + )
175 + );
176 + //error_log('[MXCHAT-DB] Checked for existing URL, found ID: ' . ($existing_id ?: 'none'));
177 + } else {
178 + //error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)');
179 + }
180 + // ===== END FIX =====
181 +
182 + // Progressive fallback mechanism for problematic content
183 + $attempt = 1;
184 + $max_attempts = 3;
185 + $current_content = $safe_content;
186 + $result = false;
187 +
188 + while ($attempt <= $max_attempts && $result === false) {
189 + try {
190 + if ($existing_id) {
191 + //error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
192 +
193 + // Update the existing row
194 + $result = $wpdb->update(
195 + $table_name,
196 + array(
197 + 'url' => $source_url,
198 + 'article_content' => $current_content,
199 + 'embedding_vector' => $embedding_vector_serialized,
200 + 'source_url' => $source_url,
201 + 'timestamp' => current_time('mysql'),
202 + ),
203 + array('id' => $existing_id),
204 + array('%s','%s','%s','%s','%s'),
205 + array('%d')
206 + );
207 + } else {
208 + //error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
209 + //error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
210 +
211 + // Insert a new row (using generated unique ID for manual content)
212 + $result = $wpdb->insert(
213 + $table_name,
214 + array(
215 + 'url' => $source_url, // Now unique for manual content
216 + 'article_content' => $current_content,
217 + 'embedding_vector' => $embedding_vector_serialized,
218 + 'source_url' => $source_url, // Now unique for manual content
219 + 'timestamp' => current_time('mysql'),
220 + ),
221 + array('%s','%s','%s','%s','%s')
222 + );
223 + }
224 +
225 + if ($result === false) {
226 + //error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . '): ' . $wpdb->last_error);
227 +
228 + // Progressively apply more aggressive sanitization on failure
229 + if ($attempt === 1) {
230 + // First fallback: Use a more aggressive character filter and shorten
231 + $current_content = preg_replace('/[^\p{L}\p{N}\s.,;:!?()-]/u', '', $current_content);
232 + $current_content = substr($current_content, 0, 50000);
233 + } else if ($attempt === 2) {
234 + // Second fallback: Keep only alphanumeric and basic punctuation, shorten further
235 + $current_content = preg_replace('/[^a-zA-Z0-9\s.,;:!?()-]/u', '', $current_content);
236 + $current_content = substr($current_content, 0, 30000);
237 + }
238 +
239 + $attempt++;
240 + }
241 + } catch (Exception $e) {
242 + //error_log('[MXCHAT-DB] Exception during database operation: ' . $e->getMessage());
243 + $attempt++;
244 + }
245 + }
246 +
247 + if ($result === false) {
248 + //error_log('[MXCHAT-DB] All database operation attempts failed');
249 + return new WP_Error('database_failed', 'Failed to store content in WordPress database after ' . $max_attempts . ' attempts');
250 + }
251 +
252 + //error_log('[MXCHAT-DB] WordPress database operation completed successfully (Attempt ' . ($attempt - 1) . ')');
253 + return true;
254 +}
255 +
256 +/**
257 + * UPDATED: Store content in Pinecone database with bot support
258 + */
259 +private static function store_in_pinecone_main($embedding_vector, $content, $url, $api_key, $environment, $index_name, $vector_id = null, $bot_id = 'default', $namespace = '') {
260 + //error_log('[MXCHAT-PINECONE-MAIN] ===== Starting Pinecone storage for bot ' . $bot_id . ' =====');
261 +
262 + // ===== UPDATED: Handle manual content with unique vector IDs =====
263 + if ($vector_id) {
264 + // Use provided vector ID
265 + //error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
266 + } elseif (!empty($url) && filter_var($url, FILTER_VALIDATE_URL)) {
267 + // For valid URLs, use URL-based ID (existing behavior)
268 + $vector_id = md5($url);
269 + //error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
270 + } else {
271 + // For manual content (empty/invalid URL), generate unique ID
272 + $vector_id = 'manual_' . time() . '_' . substr(md5($content . microtime(true)), 0, 8);
273 + //error_log('[MXCHAT-PINECONE-MAIN] Generated unique vector ID for manual content: ' . $vector_id);
274 + }
275 + // ===== END UPDATE =====
276 +
277 + // Get host from bot-specific config or fallback to default
278 + if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
279 + $options = get_option('mxchat_pinecone_addon_options');
280 + $host = $options['mxchat_pinecone_host'] ?? '';
281 + } else {
282 + $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
283 + if (!empty($bot_pinecone_config)) {
284 + $host = $bot_pinecone_config['host'] ?? '';
285 + } else {
286 + $options = get_option('mxchat_pinecone_addon_options');
287 + $host = $options['mxchat_pinecone_host'] ?? '';
288 + }
289 + }
290 +
291 + //error_log('[MXCHAT-PINECONE-MAIN] Host: ' . $host);
292 + //error_log('[MXCHAT-PINECONE-MAIN] API key length: ' . strlen($api_key));
293 + //error_log('[MXCHAT-PINECONE-MAIN] Bot ID: ' . $bot_id);
294 + //error_log('[MXCHAT-PINECONE-MAIN] Namespace: ' . $namespace);
295 +
296 + if (empty($host)) {
297 + //error_log('[MXCHAT-PINECONE-MAIN] ERROR: Host is empty');
298 + return new WP_Error('pinecone_config', 'Pinecone host is not configured. Please set the host in your bot settings.');
299 + }
300 +
301 + // ===== UPDATED: Determine content type more accurately =====
302 + $is_product = false;
303 + $content_type = 'manual'; // Default for manual content
304 +
305 + if (!empty($url) && filter_var($url, FILTER_VALIDATE_URL)) {
306 + $is_product = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
307 + $content_type = $is_product ? 'product' : 'content';
308 + }
309 +
310 + //error_log('[MXCHAT-PINECONE-MAIN] Content type: ' . $content_type);
311 + // ===== END UPDATE =====
312 +
313 + $api_endpoint = "https://{$host}/vectors/upsert";
314 + //error_log('[MXCHAT-PINECONE-MAIN] API endpoint: ' . $api_endpoint);
315 +
316 + // UPDATED: Add bot_id to metadata and handle namespace
317 + $metadata = array(
318 + 'text' => $content,
319 + 'source_url' => $url, // Can be empty for manual content
320 + 'type' => $content_type, // 'manual', 'content', or 'product'
321 + 'last_updated' => time(),
322 + 'created_at' => time(), // Add creation timestamp
323 + 'bot_id' => $bot_id // Add bot identification
324 + );
325 +
326 + $vector_data = array(
327 + 'id' => $vector_id,
328 + 'values' => $embedding_vector,
329 + 'metadata' => $metadata
330 + );
331 +
332 + $request_body = array(
333 + 'vectors' => array($vector_data)
334 + );
335 +
336 + // Add namespace if specified for multi-bot separation
337 + if (!empty($namespace)) {
338 + $request_body['namespace'] = $namespace;
339 + //error_log('[MXCHAT-PINECONE-MAIN] Using namespace: ' . $namespace);
340 + }
341 +
342 + //error_log('[MXCHAT-PINECONE-MAIN] Request body prepared (embedding dimensions: ' . count($embedding_vector) . ')');
343 +
344 + $response = wp_remote_post($api_endpoint, array(
345 + 'headers' => array(
346 + 'Api-Key' => $api_key,
347 + 'accept' => 'application/json',
348 + 'content-type' => 'application/json'
349 + ),
350 + 'body' => wp_json_encode($request_body),
351 + 'timeout' => 30,
352 + 'data_format' => 'body'
353 + ));
354 +
355 + if (is_wp_error($response)) {
356 + //error_log('[MXCHAT-PINECONE-MAIN] WordPress request error: ' . $response->get_error_message());
357 + return new WP_Error('pinecone_request', $response->get_error_message());
358 + }
359 +
360 + $response_code = wp_remote_retrieve_response_code($response);
361 + //error_log('[MXCHAT-PINECONE-MAIN] Response code: ' . $response_code);
362 +
363 + if ($response_code !== 200) {
364 + $body = wp_remote_retrieve_body($response);
365 + //error_log('[MXCHAT-PINECONE-MAIN] API error - Response body: ' . $body);
366 + return new WP_Error('pinecone_api', sprintf(
367 + 'Pinecone API error (HTTP %d): %s',
368 + $response_code,
369 + $body
370 + ));
371 + }
372 +
373 + $response_body = wp_remote_retrieve_body($response);
374 + //error_log('[MXCHAT-PINECONE-MAIN] Success response: ' . $response_body);
375 + //error_log('[MXCHAT-PINECONE-MAIN] Successfully stored in Pinecone for bot ' . $bot_id);
376 + //error_log('[MXCHAT-PINECONE-MAIN] ===== Pinecone storage complete =====');
377 +
378 + return true;
379 +}
380 +
381 +/**
382 + * UPDATED: Generate an embedding for the given text using bot-specific configuration.
383 + *
384 + * @param string $text The text to be embedded.
385 + * @param string $api_key The API key used for generating embeddings.
386 + * @param string $bot_id The bot ID for multi-bot support
387 + * @return array|null The embedding vector or null on failure.
388 + */
389 +private static function generate_embedding($text, $api_key, $bot_id = 'default') {
390 + // Get bot-specific options
391 + if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
392 + $options = get_option('mxchat_options');
393 + } else {
394 + $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
395 + $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
396 + }
397 +
398 + $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
399 +
400 + // Determine endpoint and API key based on model
401 + if (strpos($selected_model, 'voyage') === 0) {
402 + $endpoint = 'https://api.voyageai.com/v1/embeddings';
403 + $api_key = $options['voyage_api_key'] ?? '';
404 + } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
405 + $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
406 + $api_key = $options['gemini_api_key'] ?? '';
407 + } else {
408 + $endpoint = 'https://api.openai.com/v1/embeddings';
409 + // Use the bot-specific API key or fallback to passed API key
410 + $api_key = $options['api_key'] ?? $api_key;
411 + }
412 +
413 + // Prepare request body based on provider
414 + if (strpos($selected_model, 'gemini-embedding') === 0) {
415 + // Gemini API format
416 + $request_body = [
417 + 'model' => 'models/' . $selected_model,
418 + 'content' => [
419 + 'parts' => [
420 + ['text' => $text]
421 + ]
422 + ],
423 + 'outputDimensionality' => 1536
424 + ];
425 +
426 + // Prepare headers for Gemini (API key as query parameter)
427 + $endpoint .= '?key=' . $api_key;
428 + $headers = [
429 + 'Content-Type' => 'application/json'
430 + ];
431 + } else {
432 + // OpenAI/Voyage API format
433 + $request_body = [
434 + 'input' => $text,
435 + 'model' => $selected_model
436 + ];
437 +
438 + // Add output_dimension for voyage-3-large
439 + if ($selected_model === 'voyage-3-large') {
440 + $request_body['output_dimension'] = 2048;
441 + }
442 +
443 + // Prepare headers for OpenAI/Voyage
444 + $headers = [
445 + 'Content-Type' => 'application/json',
446 + 'Authorization' => 'Bearer ' . $api_key
447 + ];
448 + }
449 +
450 + $args = [
451 + 'body' => wp_json_encode($request_body),
452 + 'headers' => $headers,
453 + 'timeout' => 60,
454 + 'redirection' => 5,
455 + 'blocking' => true,
456 + 'httpversion' => '1.0',
457 + 'sslverify' => true,
458 + ];
459 +
460 + $response = wp_remote_post($endpoint, $args);
461 +
462 + if (is_wp_error($response)) {
463 + //error_log('Error generating embedding for bot ' . $bot_id . ': ' . $response->get_error_message());
464 + return null;
465 + }
466 +
467 + $response_body = json_decode(wp_remote_retrieve_body($response), true);
468 +
469 + // Handle different response formats based on provider
470 + if (strpos($selected_model, 'gemini-embedding') === 0) {
471 + // Gemini API response format
472 + if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
473 + return $response_body['embedding']['values'];
474 + } else {
475 + //error_log('Invalid response received from Gemini embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
476 + return null;
477 + }
478 + } else {
479 + // OpenAI/Voyage API response format
480 + if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
481 + return $response_body['data'][0]['embedding'];
482 + } else {
483 + //error_log('Invalid response received from embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
484 + return null;
485 + }
486 + }
487 +}
1068 488 }