PluginProbe
MxChat – AI Chatbot & Content Generation for WordPress / 2.4.6
MxChat – AI Chatbot & Content Generation for WordPress v2.4.6
3.2.21 3.2.20 3.2.19 3.2.18 3.2.17 3.2.16 3.2.15 3.2.14 3.2.12 3.2.13 3.2.11 3.2.10 3.2.9 3.2.8 3.2.7 3.2.6 3.2.5 3.2.4 3.2.3 3.2.2 3.2.1 2.0.3 2.0.4 2.0.5 2.0.6 All 152 releases
← All changes | includes/class-mxchat-utils.php +487 -1097 3.2.15 → 2.4.6 View file →
@@ -1,1098 +1,488 @@
1 -<?php
2 -if (!defined('ABSPATH')) {
3 - exit; // Exit if accessed directly
4 -}
5 -
6 -class MxChat_Utils {
7 -
8 -/**
9 - * Centralized embedding model registry. Single source of truth for dimensions
10 - * and provider, so model-switch protection logic doesn't drift across files.
11 - */
12 -public static function embedding_model_registry() {
13 - return array(
14 - 'text-embedding-ada-002' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'Ada 2'),
15 - 'text-embedding-3-small' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'TE3 Small'),
16 - 'text-embedding-3-large' => array('dims' => 3072, 'provider' => 'openai', 'label' => 'TE3 Large'),
17 - 'voyage-3-large' => array('dims' => 2048, 'provider' => 'voyage', 'label' => 'Voyage-3 Large'),
18 - 'gemini-embedding-001' => array('dims' => 1536, 'provider' => 'gemini', 'label' => 'Gemini Embedding'),
19 - );
20 -}
21 -
22 -public static function embedding_model_dimensions($model) {
23 - $registry = self::embedding_model_registry();
24 - return isset($registry[$model]) ? (int) $registry[$model]['dims'] : 0;
25 -}
26 -
27 -public static function embedding_model_label($model) {
28 - $registry = self::embedding_model_registry();
29 - return isset($registry[$model]) ? $registry[$model]['label'] : $model;
30 -}
31 -
32 -/**
33 - * Returns the model that was last used to actually write embeddings into the
34 - * KB. Differs from the user-selected setting once a switch has happened but
35 - * no re-embed has occurred yet — that's the mismatch state we warn about.
36 - */
37 -public static function get_active_embedding_model() {
38 - return get_option('mxchat_active_embedding_model', '');
39 -}
40 -
41 -/**
42 - * Stamp the model that produced the most recent successful embedding. Called
43 - * from generate_embedding() right after the API responds with a valid vector.
44 - */
45 -public static function stamp_active_embedding_model($model) {
46 - if (!empty($model) && $model !== self::get_active_embedding_model()) {
47 - update_option('mxchat_active_embedding_model', $model, false);
48 - }
49 -}
50 -
51 -/**
52 - * Extract the 11-character YouTube video ID from a URL, or '' if the URL is
53 - * not a single-video YouTube link. Single source of truth for both the KB
54 - * ingestion side and the chat render side — do not duplicate this parsing.
55 - * Channel, playlist, and search URLs deliberately return '' (only a URL that
56 - * identifies one video can be embedded).
57 - */
58 -public static function parse_youtube_id($url) {
59 - if (!is_string($url) || $url === '' || !preg_match('#^https?://#i', $url)) {
60 - return '';
61 - }
62 - $host = strtolower((string) wp_parse_url($url, PHP_URL_HOST));
63 - $host = preg_replace('/^(www|m)\./', '', $host);
64 - $path = (string) wp_parse_url($url, PHP_URL_PATH);
65 - $id = '';
66 - if ($host === 'youtu.be') {
67 - $segments = explode('/', ltrim($path, '/'));
68 - $id = $segments[0] ?? '';
69 - } elseif (in_array($host, array('youtube.com', 'youtube-nocookie.com'), true)) {
70 - if (preg_match('#^/(?:shorts|embed|live|v)/([A-Za-z0-9_-]+)#', $path, $m)) {
71 - $id = $m[1];
72 - } elseif ($path === '/watch') {
73 - parse_str((string) wp_parse_url($url, PHP_URL_QUERY), $query_vars);
74 - $id = isset($query_vars['v']) ? (string) $query_vars['v'] : '';
75 - }
76 - }
77 - $id = preg_replace('/[^A-Za-z0-9_-]/', '', (string) $id);
78 - return (strlen($id) === 11) ? $id : '';
79 -}
80 -
81 -/**
82 - * UPDATED: Submit or update content (and its embedding) in the database.
83 - * Stores in Pinecone if enabled, otherwise stores in WordPress DB.
84 - *
85 - * @param string $content The content to be embedded.
86 - * @param string $source_url The source URL of the content.
87 - * @param string $api_key The API key used for generating embeddings.
88 - * @param string $vector_id Optional vector ID for Pinecone (if not provided, will use md5 of URL)
89 - * @param string $bot_id The bot ID for multi-bot support
90 - * @param string $content_type The type of content (post, page, pdf, url, manual, product, etc.)
91 - * @return bool|WP_Error True on success, WP_Error on failure
92 - */
93 -public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
94 - global $wpdb;
95 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
96 -
97 - //error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ', Type: ' . $content_type . ')');
98 - //error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
99 -
100 - // Sanitize the source URL
101 - $source_url = esc_url_raw($source_url);
102 -
103 - // Sanitize content_type
104 - $content_type = sanitize_key($content_type);
105 - if (empty($content_type)) {
106 - $content_type = 'content'; // Fallback for backwards compatibility
107 - }
108 -
109 - // Just ensure UTF-8 validity without aggressive escaping
110 - $safe_content = wp_check_invalid_utf8($content);
111 - // Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D)
112 - $safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
113 -
114 - // Check if chunking should be applied
115 - $chunker = MxChat_Chunker::from_settings();
116 - if ($chunker->should_chunk($safe_content)) {
117 - //error_log('[MXCHAT-DB] Content exceeds chunk threshold, using chunked submission');
118 - return self::submit_chunked_content($safe_content, $source_url, $api_key, $bot_id, $content_type, $chunker);
119 - }
120 -
121 - // UPDATED: Generate the embedding using bot-specific configuration
122 - $embedding_vector = self::generate_embedding($content, $api_key, $bot_id);
123 -
124 - if (!is_array($embedding_vector)) {
125 - //error_log('[MXCHAT-DB] Error: Embedding generation failed');
126 - return new WP_Error('embedding_failed', 'Failed to generate embedding for content');
127 - }
128 -
129 - //error_log('[MXCHAT-DB] Embedding generated successfully');
130 -
131 - // UPDATED: Check if Pinecone is enabled for this specific bot
132 - if (self::is_pinecone_enabled_for_bot($bot_id)) {
133 - //error_log('[MXCHAT-DB] Pinecone is enabled for bot ' . $bot_id . ' - using Pinecone storage');
134 - // Store in Pinecone only
135 - return self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id, $content_type);
136 - } else {
137 - //error_log('[MXCHAT-DB] Pinecone not enabled for bot ' . $bot_id . ' - using WordPress storage');
138 - // Store in WordPress database only
139 - $embedding_vector_serialized = maybe_serialize($embedding_vector);
140 - return self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type);
141 - }
142 -}
143 -
144 -/**
145 - * UPDATED: Check if Pinecone is enabled and properly configured for a specific bot
146 - */
147 -private static function is_pinecone_enabled_for_bot($bot_id = 'default') {
148 - // For default bot or when multi-bot is not active, use original method
149 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
150 - return self::is_pinecone_enabled();
151 - }
152 -
153 - // Get bot-specific Pinecone configuration
154 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
155 -
156 - if (empty($bot_pinecone_config)) {
157 - // Fallback to default configuration
158 - return self::is_pinecone_enabled();
159 - }
160 -
161 - $enabled_check = !empty($bot_pinecone_config['use_pinecone']) && $bot_pinecone_config['use_pinecone'];
162 - $api_key_check = !empty($bot_pinecone_config['api_key']);
163 - $host_check = !empty($bot_pinecone_config['host']);
164 -
165 - return $enabled_check && $api_key_check && $host_check;
166 -}
167 -
168 -/**
169 - * Check if Pinecone is enabled and properly configured (original method for default bot)
170 - */
171 -private static function is_pinecone_enabled() {
172 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
173 -
174 - if (empty($pinecone_options)) {
175 - return false;
176 - }
177 -
178 - $enabled_check = !empty($pinecone_options['mxchat_use_pinecone']) && $pinecone_options['mxchat_use_pinecone'] !== '0';
179 - $api_key_check = !empty($pinecone_options['mxchat_pinecone_api_key']);
180 - $host_check = !empty($pinecone_options['mxchat_pinecone_host']);
181 -
182 - return $enabled_check && $api_key_check && $host_check;
183 -}
184 -
185 -/**
186 - * UPDATED: Store content in Pinecone only with bot support
187 - */
188 -private static function store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
189 - //error_log('[MXCHAT-PINECONE] ===== Using Pinecone-only storage for bot ' . $bot_id . ' =====');
190 -
191 - // Get bot-specific Pinecone configuration
192 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
193 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
194 - $api_key = $pinecone_options['mxchat_pinecone_api_key'];
195 - $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
196 - $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
197 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
198 - } else {
199 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
200 - if (empty($bot_pinecone_config)) {
201 - // Fallback to default configuration
202 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
203 - $api_key = $pinecone_options['mxchat_pinecone_api_key'];
204 - $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
205 - $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
206 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
207 - } else {
208 - $api_key = $bot_pinecone_config['api_key'];
209 - $environment = ''; // Not used in new Pinecone API
210 - $index_name = ''; // Not used in new Pinecone API
211 - $namespace = $bot_pinecone_config['namespace'] ?? '';
212 - }
213 - }
214 -
215 - $result = self::store_in_pinecone_main(
216 - $embedding_vector,
217 - $content,
218 - $source_url,
219 - $api_key,
220 - $environment,
221 - $index_name,
222 - $vector_id,
223 - $bot_id,
224 - $namespace,
225 - $content_type
226 - );
227 -
228 - if (is_wp_error($result)) {
229 - //error_log('[MXCHAT-PINECONE] Pinecone storage failed for bot ' . $bot_id . ': ' . $result->get_error_message());
230 - return $result;
231 - }
232 -
233 - //error_log('[MXCHAT-PINECONE] Pinecone storage completed successfully for bot ' . $bot_id);
234 - return true;
235 -}
236 -
237 -/**
238 - * Store content in WordPress database with progressive fallback
239 - * UPDATED 2.5.6: Now includes content_type parameter
240 - */
241 -private static function store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type = 'content') {
242 - global $wpdb;
243 -
244 - //error_log('[MXCHAT-DB] ===== Using WordPress-only storage =====');
245 -
246 - // Sanitize content_type
247 - $content_type = sanitize_key($content_type);
248 - if (empty($content_type)) {
249 - $content_type = 'content'; // Fallback for backwards compatibility
250 - }
251 -
252 - // ===== FIXED: Generate unique identifier for manual content =====
253 - $original_source_url = $source_url;
254 - // Check if this is truly manual content (no URL at all) vs a real URL that filter_var rejects
255 - // filter_var(FILTER_VALIDATE_URL) rejects valid URLs with encoded chars, non-ASCII, fragments, etc.
256 - // Use a looser check: if it starts with http(s):// or has a scheme, it's a URL
257 - $has_url_scheme = !empty($source_url) && preg_match('#^https?://#i', $source_url);
258 - // Treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
259 - $is_legacy_mxchat_url = $has_url_scheme && strpos($source_url, 'mxchat.ai') !== false;
260 - $is_manual_content = empty($source_url) || $source_url === '' || !$has_url_scheme || $is_legacy_mxchat_url;
261 -
262 - if ($is_manual_content) {
263 - // Generate unique identifier for manual content to prevent overwrites
264 - $source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
265 - //error_log('[MXCHAT-DB] Generated unique ID for manual content: ' . $source_url);
266 - }
267 -
268 - // Only check for duplicates if we have a valid source URL (not manual content)
269 - $existing_id = null;
270 - if (!$is_manual_content) {
271 - $existing_id = $wpdb->get_var(
272 - $wpdb->prepare(
273 - "SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
274 - $source_url
275 - )
276 - );
277 - //error_log('[MXCHAT-DB] Checked for existing URL, found ID: ' . ($existing_id ?: 'none'));
278 - } else {
279 - //error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)');
280 - }
281 - // ===== END FIX =====
282 -
283 - // Progressive fallback mechanism for problematic content
284 - $attempt = 1;
285 - $max_attempts = 3;
286 - $current_content = $safe_content;
287 - $result = false;
288 -
289 - while ($attempt <= $max_attempts && $result === false) {
290 - try {
291 - if ($existing_id) {
292 - //error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
293 -
294 - // Update the existing row - UPDATED 2.5.6: Added content_type
295 - $result = $wpdb->update(
296 - $table_name,
297 - array(
298 - 'url' => $source_url,
299 - 'article_content' => $current_content,
300 - 'embedding_vector' => $embedding_vector_serialized,
301 - 'source_url' => $source_url,
302 - 'content_type' => $content_type,
303 - 'timestamp' => current_time('mysql'),
304 - ),
305 - array('id' => $existing_id),
306 - array('%s','%s','%s','%s','%s','%s'),
307 - array('%d')
308 - );
309 - } else {
310 - //error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
311 - //error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
312 -
313 - // Insert a new row - UPDATED 2.5.6: Added content_type
314 - $result = $wpdb->insert(
315 - $table_name,
316 - array(
317 - 'url' => $source_url, // Now unique for manual content
318 - 'article_content' => $current_content,
319 - 'embedding_vector' => $embedding_vector_serialized,
320 - 'source_url' => $source_url, // Now unique for manual content
321 - 'content_type' => $content_type,
322 - 'timestamp' => current_time('mysql'),
323 - ),
324 - array('%s','%s','%s','%s','%s','%s')
325 - );
326 - }
327 -
328 - if ($result === false) {
329 - //error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . ')');
330 - //error_log('[MXCHAT-DB] MySQL Error: ' . $wpdb->last_error);
331 - //error_log('[MXCHAT-DB] MySQL Error Number: ' . $wpdb->last_errno);
332 - //error_log('[MXCHAT-DB] Last Query: ' . substr($wpdb->last_query, 0, 500));
333 - //error_log('[MXCHAT-DB] Content length: ' . strlen($current_content) . ' bytes');
334 - //error_log('[MXCHAT-DB] Embedding vector length: ' . strlen($embedding_vector_serialized) . ' bytes');
335 -
336 - // Progressively apply more aggressive sanitization on failure
337 - if ($attempt === 1) {
338 - // First fallback: Use a more aggressive character filter and shorten
339 - $current_content = preg_replace('/[^\p{L}\p{N}\s.,;:!?()-]/u', '', $current_content);
340 - $current_content = substr($current_content, 0, 50000);
341 - } else if ($attempt === 2) {
342 - // Second fallback: Keep only alphanumeric and basic punctuation, shorten further
343 - $current_content = preg_replace('/[^a-zA-Z0-9\s.,;:!?()-]/u', '', $current_content);
344 - $current_content = substr($current_content, 0, 30000);
345 - }
346 -
347 - $attempt++;
348 - }
349 - } catch (Exception $e) {
350 - //error_log('[MXCHAT-DB] Exception during database operation: ' . $e->getMessage());
351 - $attempt++;
352 - }
353 - }
354 -
355 -if ($result === false) {
356 - //error_log('[MXCHAT-DB] All database operation attempts failed');
357 - //error_log('[MXCHAT-DB] Final MySQL Error: ' . $wpdb->last_error);
358 -
359 - $detailed_error = sprintf(
360 - 'Failed to store content in WordPress database after %d attempts. MySQL Error: %s (Error #%d). Content size: %d bytes, Embedding size: %d bytes',
361 - $max_attempts,
362 - $wpdb->last_error,
363 - $wpdb->last_errno,
364 - strlen($current_content),
365 - strlen($embedding_vector_serialized)
366 - );
367 -
368 - return new WP_Error('database_failed', $detailed_error);
369 -}
370 -
371 - //error_log('[MXCHAT-DB] WordPress database operation completed successfully (Attempt ' . ($attempt - 1) . ')');
372 - return true;
373 -}
374 -
375 -/**
376 - * UPDATED: Store content in Pinecone database with bot support
377 - * UPDATED 2.5.6: Now accepts content_type parameter
378 - */
379 -private static function store_in_pinecone_main($embedding_vector, $content, $url, $api_key, $environment, $index_name, $vector_id = null, $bot_id = 'default', $namespace = '', $content_type = 'content') {
380 - //error_log('[MXCHAT-PINECONE-MAIN] ===== Starting Pinecone storage for bot ' . $bot_id . ' =====');
381 -
382 - // ===== UPDATED: Handle manual content with unique vector IDs =====
383 - if ($vector_id) {
384 - // Use provided vector ID
385 - //error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
386 - } elseif (!empty($url) && preg_match('#^https?://#i', $url)) {
387 - // For URLs, use URL-based ID (existing behavior)
388 - $vector_id = md5($url);
389 - //error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
390 - } else {
391 - // For manual content (empty/no URL scheme), generate unique ID
392 - $vector_id = 'manual_' . time() . '_' . substr(md5($content . microtime(true)), 0, 8);
393 - //error_log('[MXCHAT-PINECONE-MAIN] Generated unique vector ID for manual content: ' . $vector_id);
394 - }
395 - // ===== END UPDATE =====
396 -
397 - // Get host from bot-specific config or fallback to default
398 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
399 - $options = get_option('mxchat_pinecone_addon_options');
400 - $host = $options['mxchat_pinecone_host'] ?? '';
401 - } else {
402 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
403 - if (!empty($bot_pinecone_config)) {
404 - $host = $bot_pinecone_config['host'] ?? '';
405 - } else {
406 - $options = get_option('mxchat_pinecone_addon_options');
407 - $host = $options['mxchat_pinecone_host'] ?? '';
408 - }
409 - }
410 -
411 - //error_log('[MXCHAT-PINECONE-MAIN] Host: ' . $host);
412 - //error_log('[MXCHAT-PINECONE-MAIN] API key length: ' . strlen($api_key));
413 - //error_log('[MXCHAT-PINECONE-MAIN] Bot ID: ' . $bot_id);
414 - //error_log('[MXCHAT-PINECONE-MAIN] Namespace: ' . $namespace);
415 -
416 - if (empty($host)) {
417 - //error_log('[MXCHAT-PINECONE-MAIN] ERROR: Host is empty');
418 - return new WP_Error('pinecone_config', 'Pinecone host is not configured. Please set the host in your bot settings.');
419 - }
420 -
421 - // ===== UPDATED 2.5.6: Use passed content_type or determine from URL if not provided =====
422 - // Sanitize content_type
423 - $content_type = sanitize_key($content_type);
424 - if (empty($content_type)) {
425 - // Fallback to old detection logic for backwards compatibility
426 - $is_product = false;
427 - $content_type = 'manual'; // Default for manual content
428 -
429 - if (!empty($url) && preg_match('#^https?://#i', $url)) {
430 - $is_product = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
431 - $content_type = $is_product ? 'product' : 'content';
432 - }
433 - }
434 -
435 - //error_log('[MXCHAT-PINECONE-MAIN] Content type: ' . $content_type);
436 - // ===== END UPDATE =====
437 -
438 - $api_endpoint = "https://{$host}/vectors/upsert";
439 - //error_log('[MXCHAT-PINECONE-MAIN] API endpoint: ' . $api_endpoint);
440 -
441 - // UPDATED 2.5.6: Use provided content_type in metadata
442 - $metadata = array(
443 - 'text' => $content,
444 - 'source_url' => $url, // Can be empty for manual content
445 - 'type' => $content_type, // Now supports: post, page, pdf, url, manual, product, etc.
446 - 'last_updated' => time(),
447 - 'created_at' => time(), // Add creation timestamp
448 - 'bot_id' => $bot_id, // Add bot identification
449 - );
450 -
451 - $vector_data = array(
452 - 'id' => $vector_id,
453 - 'values' => $embedding_vector,
454 - 'metadata' => $metadata
455 - );
456 -
457 - $request_body = array(
458 - 'vectors' => array($vector_data)
459 - );
460 -
461 - // Add namespace if specified for multi-bot separation
462 - if (!empty($namespace)) {
463 - $request_body['namespace'] = $namespace;
464 - //error_log('[MXCHAT-PINECONE-MAIN] Using namespace: ' . $namespace);
465 - }
466 -
467 - //error_log('[MXCHAT-PINECONE-MAIN] Request body prepared (embedding dimensions: ' . count($embedding_vector) . ')');
468 -
469 - $response = wp_remote_post($api_endpoint, array(
470 - 'headers' => array(
471 - 'Api-Key' => $api_key,
472 - 'accept' => 'application/json',
473 - 'content-type' => 'application/json'
474 - ),
475 - 'body' => wp_json_encode($request_body),
476 - 'timeout' => 30,
477 - 'data_format' => 'body'
478 - ));
479 -
480 - if (is_wp_error($response)) {
481 - //error_log('[MXCHAT-PINECONE-MAIN] WordPress request error: ' . $response->get_error_message());
482 - return new WP_Error('pinecone_request', $response->get_error_message());
483 - }
484 -
485 - $response_code = wp_remote_retrieve_response_code($response);
486 - //error_log('[MXCHAT-PINECONE-MAIN] Response code: ' . $response_code);
487 -
488 - if ($response_code !== 200) {
489 - $body = wp_remote_retrieve_body($response);
490 - //error_log('[MXCHAT-PINECONE-MAIN] API error - Response body: ' . $body);
491 - return new WP_Error('pinecone_api', sprintf(
492 - 'Pinecone API error (HTTP %d): %s',
493 - $response_code,
494 - $body
495 - ));
496 - }
497 -
498 - $response_body = wp_remote_retrieve_body($response);
499 - //error_log('[MXCHAT-PINECONE-MAIN] Success response: ' . $response_body);
500 - //error_log('[MXCHAT-PINECONE-MAIN] Successfully stored in Pinecone for bot ' . $bot_id);
501 - //error_log('[MXCHAT-PINECONE-MAIN] ===== Pinecone storage complete =====');
502 -
503 - return true;
504 -}
505 -
506 -/**
507 - * UPDATED: Generate an embedding for the given text using bot-specific configuration.
508 - *
509 - * @param string $text The text to be embedded.
510 - * @param string $api_key The API key used for generating embeddings.
511 - * @param string $bot_id The bot ID for multi-bot support
512 - * @return array|null The embedding vector or null on failure.
513 - */
514 -private static function generate_embedding($text, $api_key, $bot_id = 'default') {
515 - // Get bot-specific options
516 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
517 - $options = get_option('mxchat_options');
518 - } else {
519 - $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
520 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
521 - }
522 -
523 - // Opt-in: when the custom provider is selected for embeddings, route the KB
524 - // INDEX side through the same custom endpoint the query side uses, so stored
525 - // vectors and query vectors come from the same model. Default-off behavior
526 - // below is untouched.
527 - if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
528 - $custom = self::generate_embedding_custom($text, $options);
529 - return is_array($custom) ? $custom : null;
530 - }
531 -
532 - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
533 -
534 - // Determine endpoint and API key based on model
535 - if (strpos($selected_model, 'voyage') === 0) {
536 - $endpoint = 'https://api.voyageai.com/v1/embeddings';
537 - $api_key = $options['voyage_api_key'] ?? '';
538 - } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
539 - $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
540 - $api_key = $options['gemini_api_key'] ?? '';
541 - } else {
542 - $endpoint = 'https://api.openai.com/v1/embeddings';
543 - // Use the bot-specific API key or fallback to passed API key
544 - $api_key = $options['api_key'] ?? $api_key;
545 - }
546 -
547 - // Prepare request body based on provider
548 - if (strpos($selected_model, 'gemini-embedding') === 0) {
549 - // Gemini API format
550 - $request_body = [
551 - 'model' => 'models/' . $selected_model,
552 - 'content' => [
553 - 'parts' => [
554 - ['text' => $text]
555 - ]
556 - ],
557 - 'outputDimensionality' => 1536
558 - ];
559 -
560 - // Prepare headers for Gemini (API key as query parameter)
561 - $endpoint .= '?key=' . $api_key;
562 - $headers = [
563 - 'Content-Type' => 'application/json'
564 - ];
565 - } else {
566 - // OpenAI/Voyage API format
567 - $request_body = [
568 - 'input' => $text,
569 - 'model' => $selected_model
570 - ];
571 -
572 - // Add output_dimension for voyage-3-large
573 - if ($selected_model === 'voyage-3-large') {
574 - $request_body['output_dimension'] = 2048;
575 - }
576 -
577 - // Prepare headers for OpenAI/Voyage
578 - $headers = [
579 - 'Content-Type' => 'application/json',
580 - 'Authorization' => 'Bearer ' . $api_key
581 - ];
582 - }
583 -
584 - $args = [
585 - 'body' => wp_json_encode($request_body),
586 - 'headers' => $headers,
587 - 'timeout' => 60,
588 - 'redirection' => 5,
589 - 'blocking' => true,
590 - 'httpversion' => '1.0',
591 - 'sslverify' => true,
592 - ];
593 -
594 - $response = wp_remote_post($endpoint, $args);
595 -
596 - if (is_wp_error($response)) {
597 - //error_log('Error generating embedding for bot ' . $bot_id . ': ' . $response->get_error_message());
598 - return null;
599 - }
600 -
601 - $response_body = json_decode(wp_remote_retrieve_body($response), true);
602 -
603 - // Handle different response formats based on provider
604 - if (strpos($selected_model, 'gemini-embedding') === 0) {
605 - // Gemini API response format
606 - if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
607 - self::stamp_active_embedding_model($selected_model);
608 - return $response_body['embedding']['values'];
609 - } else {
610 - //error_log('Invalid response received from Gemini embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
611 - return null;
612 - }
613 - } else {
614 - // OpenAI/Voyage API response format
615 - if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
616 - self::stamp_active_embedding_model($selected_model);
617 - return $response_body['data'][0]['embedding'];
618 - } else {
619 - //error_log('Invalid response received from embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
620 - return null;
621 - }
622 - }
623 -}
624 -
625 -/**
626 - * Generate an embedding via a Custom (OpenAI-compatible) provider's /embeddings route.
627 - * Shared by every embedding entry point so the KNOWLEDGE-BASE INDEX side and the
628 - * QUERY side route through the same model when the opt-in
629 - * 'custom_provider_for_embeddings' setting is on. Mirrors the query-path logic in
630 - * MxChat_Integrator::mxchat_generate_embedding_custom() but takes an explicit
631 - * $options array so it is callable statically from utils + knowledge-manager.
632 - *
633 - * Returns a numeric array (the embedding vector) on success, or a human-readable
634 - * error string on failure (so callers expecting a string error, like the
635 - * knowledge-manager, can surface it directly; callers expecting array|null wrap it).
636 - *
637 - * @param string $text Text to embed.
638 - * @param array $options The resolved mxchat options (must contain the custom_provider_* keys).
639 - * @return array|string Embedding vector on success; error string on failure.
640 - */
641 -public static function generate_embedding_custom($text, $options) {
642 - if (empty($text)) {
643 - return 'No text provided for embedding generation';
644 - }
645 -
646 - $base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : '';
647 - if (empty($base_url)) {
648 - return 'Custom provider Base URL is not configured.';
649 - }
650 -
651 - $api_key = isset($options['custom_provider_api_key']) ? trim((string) $options['custom_provider_api_key']) : '';
652 - $auth_scheme = isset($options['custom_provider_auth_scheme']) ? $options['custom_provider_auth_scheme'] : 'bearer';
653 - $api_version = isset($options['custom_provider_api_version']) ? trim((string) $options['custom_provider_api_version']) : '';
654 -
655 - // Embedding model: prefer the dedicated custom_provider_embedding_model, fall back to the chat model.
656 - $model = (isset($options['custom_provider_embedding_model']) && trim((string) $options['custom_provider_embedding_model']) !== '')
657 - ? trim((string) $options['custom_provider_embedding_model'])
658 - : ((isset($options['custom_provider_model']) && trim((string) $options['custom_provider_model']) !== '') ? trim((string) $options['custom_provider_model']) : 'default');
659 -
660 - $embed_url = $base_url . '/embeddings';
661 - if (!empty($api_version)) {
662 - $embed_url .= (strpos($embed_url, '?') === false ? '?' : '&') . 'api-version=' . rawurlencode($api_version);
663 - }
664 -
665 - $headers = ['Content-Type' => 'application/json'];
666 - if (!empty($api_key)) {
667 - if ($auth_scheme === 'api-key') {
668 - $headers['api-key'] = $api_key;
669 - } else {
670 - $headers['Authorization'] = 'Bearer ' . $api_key;
671 - }
672 - }
673 -
674 - $response = wp_remote_post($embed_url, [
675 - 'headers' => $headers,
676 - 'body' => wp_json_encode(['input' => $text, 'model' => $model]),
677 - 'timeout' => 60,
678 - ]);
679 - if (is_wp_error($response)) {
680 - return 'Connection error when generating embeddings (custom provider): ' . $response->get_error_message();
681 - }
682 -
683 - $status = wp_remote_retrieve_response_code($response);
684 - $body = json_decode(wp_remote_retrieve_body($response), true);
685 - if ($status !== 200) {
686 - $msg = isset($body['error']['message']) ? $body['error']['message'] : 'HTTP ' . $status;
687 - return 'Custom embedding endpoint error: ' . $msg;
688 - }
689 - if (isset($body['data'][0]['embedding']) && is_array($body['data'][0]['embedding'])) {
690 - // Stamp the custom model identity so the active-embedding-model mismatch
691 - // warning reflects the real (custom) model rather than the built-in setting.
692 - self::stamp_active_embedding_model('custom:' . $model);
693 - return $body['data'][0]['embedding'];
694 - }
695 - return 'Invalid embedding response from custom provider.';
696 -}
697 -
698 -/**
699 - * Submit content as multiple chunks
700 - *
701 - * Splits large content into chunks, generates embeddings for each,
702 - * and stores them with chunk metadata for later reassembly.
703 - *
704 - * @param string $content The content to chunk and store
705 - * @param string $source_url The source URL
706 - * @param string $api_key The API key for embeddings
707 - * @param string $bot_id The bot ID
708 - * @param string $content_type The content type
709 - * @param MxChat_Chunker $chunker The chunker instance
710 - * @return bool|WP_Error True on success, WP_Error on failure
711 - */
712 -private static function submit_chunked_content($content, $source_url, $api_key, $bot_id, $content_type, $chunker) {
713 - global $wpdb;
714 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
715 -
716 - //error_log('[MXCHAT-CHUNK-DEBUG] Starting chunked submission for: ' . $source_url);
717 - //error_log('[MXCHAT-CHUNK-DEBUG] Content length: ' . strlen($content) . ' chars');
718 -
719 - // First, delete any existing chunks for this URL (clean slate)
720 - $delete_result = self::delete_chunks_for_url($source_url, $bot_id);
721 - if (is_wp_error($delete_result)) {
722 - //error_log('[MXCHAT-CHUNK-DEBUG] Warning: Failed to delete existing chunks: ' . $delete_result->get_error_message());
723 - // Continue anyway - we'll overwrite with upsert
724 - }
725 -
726 - // Split content into chunks
727 - $chunks = $chunker->chunk_text($content);
728 - $total_chunks = count($chunks);
729 -
730 - //error_log('[MXCHAT-CHUNK-DEBUG] Created ' . $total_chunks . ' chunks');
731 - foreach ($chunks as $i => $chunk) {
732 - //error_log('[MXCHAT-CHUNK-DEBUG] Chunk ' . $i . ' length: ' . strlen($chunk) . ' chars, preview: ' . substr($chunk, 0, 100));
733 - }
734 -
735 - //error_log('[MXCHAT-CHUNK] Split content into ' . $total_chunks . ' chunks');
736 -
737 - if ($total_chunks === 0) {
738 - return new WP_Error('chunking_failed', 'Content could not be split into chunks');
739 - }
740 -
741 - $errors = array();
742 - $is_pinecone = self::is_pinecone_enabled_for_bot($bot_id);
743 -
744 - foreach ($chunks as $index => $chunk_text) {
745 - // Generate chunk metadata
746 - $chunk_metadata = MxChat_Chunker::create_chunk_metadata($index, $total_chunks, $source_url);
747 -
748 - // AI-Engine-style aliases so external consumers (Pinecone/Qdrant/Chroma) can rely on
749 - // a stable shorthand ('source'/'part_index'/'part_total') without parsing our internal names.
750 - $chunk_metadata['source'] = $source_url;
751 - $chunk_metadata['part_index'] = (int) $index;
752 - $chunk_metadata['part_total'] = (int) $total_chunks;
753 -
754 - /**
755 - * Filter the per-chunk metadata blob before it's written to the KB store.
756 - *
757 - * @param array $chunk_metadata Metadata array (source, part_index, part_total, chunk_index, total_chunks, source_url, parent_url_hash, document_type, ...).
758 - * @param string $chunk_text The chunk text being stored.
759 - * @param array $context ['bot_id' => string, 'content_type' => string, 'source_url' => string, 'part_index' => int, 'part_total' => int]
760 - * @return array Updated metadata array.
761 - */
762 - $chunk_metadata = apply_filters(
763 - 'mxchat_embedding_chunk_metadata',
764 - $chunk_metadata,
765 - $chunk_text,
766 - array(
767 - 'bot_id' => $bot_id,
768 - 'content_type' => $content_type,
769 - 'source_url' => $source_url,
770 - 'part_index' => (int) $index,
771 - 'part_total' => (int) $total_chunks,
772 - )
773 - );
774 -
775 - $chunk_vector_id = MxChat_Chunker::generate_chunk_vector_id($source_url, $index);
776 -
777 - //error_log('[MXCHAT-CHUNK] Processing chunk ' . ($index + 1) . '/' . $total_chunks . ' (ID: ' . $chunk_vector_id . ')');
778 -
779 - // Generate embedding for this chunk
780 - $embedding_vector = self::generate_embedding($chunk_text, $api_key, $bot_id);
781 -
782 - if (!is_array($embedding_vector)) {
783 - $errors[] = new WP_Error('embedding_failed', 'Failed to generate embedding for chunk ' . $index);
784 - //error_log('[MXCHAT-CHUNK] Failed to generate embedding for chunk ' . $index);
785 - continue;
786 - }
787 -
788 - if ($is_pinecone) {
789 - // Store in Pinecone with chunk metadata
790 - $result = self::store_chunk_in_pinecone(
791 - $embedding_vector,
792 - $chunk_text,
793 - $source_url,
794 - $chunk_vector_id,
795 - $bot_id,
796 - $content_type,
797 - $chunk_metadata
798 - );
799 - } else {
800 - // Store in WordPress DB with chunk metadata
801 - $content_with_metadata = MxChat_Chunker::format_chunk_for_storage($chunk_text, $chunk_metadata);
802 - $embedding_vector_serialized = maybe_serialize($embedding_vector);
803 -
804 - $result = self::store_chunk_in_wordpress_db(
805 - $content_with_metadata,
806 - $source_url,
807 - $embedding_vector_serialized,
808 - $table_name,
809 - $content_type,
810 - $chunk_metadata
811 - );
812 - }
813 -
814 - if (is_wp_error($result)) {
815 - $errors[] = $result;
816 - //error_log('[MXCHAT-CHUNK] Failed to store chunk ' . $index . ': ' . $result->get_error_message());
817 - }
818 - }
819 -
820 - if (count($errors) === $total_chunks) {
821 - return new WP_Error('chunking_failed', 'Failed to store any chunks');
822 - }
823 -
824 - if (!empty($errors)) {
825 - //error_log('[MXCHAT-CHUNK] Completed with ' . count($errors) . ' errors out of ' . $total_chunks . ' chunks');
826 - return new WP_Error('chunking_partial_failure',
827 - sprintf('Failed to store %d of %d chunks', count($errors), $total_chunks));
828 - }
829 -
830 - //error_log('[MXCHAT-CHUNK] Successfully stored all ' . $total_chunks . ' chunks');
831 - return true;
832 -}
833 -
834 -/**
835 - * Store a single chunk in Pinecone with chunk-specific metadata
836 - */
837 -private static function store_chunk_in_pinecone($embedding_vector, $chunk_text, $source_url, $vector_id, $bot_id, $content_type, $chunk_metadata) {
838 - // Get Pinecone configuration
839 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
840 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
841 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
842 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
843 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
844 - } else {
845 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
846 - if (empty($bot_pinecone_config)) {
847 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
848 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
849 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
850 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
851 - } else {
852 - $api_key = $bot_pinecone_config['api_key'] ?? '';
853 - $host = $bot_pinecone_config['host'] ?? '';
854 - $namespace = $bot_pinecone_config['namespace'] ?? '';
855 - }
856 - }
857 -
858 - if (empty($host) || empty($api_key)) {
859 - return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
860 - }
861 -
862 - $api_endpoint = "https://{$host}/vectors/upsert";
863 -
864 - // Build metadata with chunk information
865 - $metadata = array(
866 - 'text' => $chunk_text,
867 - 'source_url' => $source_url,
868 - 'type' => $content_type,
869 - 'is_chunked' => true,
870 - 'chunk_index' => $chunk_metadata['chunk_index'],
871 - 'total_chunks' => $chunk_metadata['total_chunks'],
872 - 'parent_url_hash' => $chunk_metadata['parent_url_hash'],
873 - 'last_updated' => time(),
874 - 'created_at' => time(),
875 - 'bot_id' => $bot_id,
876 - );
877 -
878 - $vector_data = array(
879 - 'id' => $vector_id,
880 - 'values' => $embedding_vector,
881 - 'metadata' => $metadata
882 - );
883 -
884 - $request_body = array(
885 - 'vectors' => array($vector_data)
886 - );
887 -
888 - if (!empty($namespace)) {
889 - $request_body['namespace'] = $namespace;
890 - }
891 -
892 - $response = wp_remote_post($api_endpoint, array(
893 - 'headers' => array(
894 - 'Api-Key' => $api_key,
895 - 'accept' => 'application/json',
896 - 'content-type' => 'application/json'
897 - ),
898 - 'body' => wp_json_encode($request_body),
899 - 'timeout' => 30
900 - ));
901 -
902 - if (is_wp_error($response)) {
903 - return $response;
904 - }
905 -
906 - $response_code = wp_remote_retrieve_response_code($response);
907 - if ($response_code !== 200) {
908 - return new WP_Error('pinecone_api', 'Pinecone API error: HTTP ' . $response_code);
909 - }
910 -
911 - return true;
912 -}
913 -
914 -/**
915 - * Store a single chunk in WordPress database
916 - */
917 -private static function store_chunk_in_wordpress_db($content_with_metadata, $source_url, $embedding_vector_serialized, $table_name, $content_type, $chunk_metadata) {
918 - global $wpdb;
919 -
920 - // For chunks, we always insert new rows (no duplicate checking)
921 - // The URL includes chunk info in the metadata, but source_url stays the same for grouping
922 - $result = $wpdb->insert(
923 - $table_name,
924 - array(
925 - 'url' => $source_url,
926 - 'article_content' => $content_with_metadata,
927 - 'embedding_vector' => $embedding_vector_serialized,
928 - 'source_url' => $source_url,
929 - 'content_type' => $content_type,
930 - 'timestamp' => current_time('mysql')
931 - ),
932 - array('%s', '%s', '%s', '%s', '%s', '%s')
933 - );
934 -
935 - if ($result === false) {
936 - return new WP_Error('database_failed', 'Failed to insert chunk: ' . $wpdb->last_error);
937 - }
938 -
939 - return true;
940 -}
941 -
942 -/**
943 - * Delete all chunks for a given URL
944 - *
945 - * @param string $source_url The source URL
946 - * @param string $bot_id The bot ID
947 - * @return bool|WP_Error True on success, WP_Error on failure
948 - */
949 -public static function delete_chunks_for_url($source_url, $bot_id = 'default') {
950 - //error_log('[MXCHAT-CHUNK-DELETE] Deleting chunks for URL: ' . $source_url);
951 -
952 - if (self::is_pinecone_enabled_for_bot($bot_id)) {
953 - return self::delete_pinecone_chunks_by_url($source_url, $bot_id);
954 - } else {
955 - return self::delete_wordpress_chunks_by_url($source_url);
956 - }
957 -}
958 -
959 -/**
960 - * Delete all chunks for a URL from Pinecone
961 - */
962 -private static function delete_pinecone_chunks_by_url($source_url, $bot_id) {
963 - // Get Pinecone configuration
964 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
965 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
966 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
967 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
968 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
969 - } else {
970 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
971 - if (empty($bot_pinecone_config)) {
972 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
973 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
974 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
975 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
976 - } else {
977 - $api_key = $bot_pinecone_config['api_key'] ?? '';
978 - $host = $bot_pinecone_config['host'] ?? '';
979 - $namespace = $bot_pinecone_config['namespace'] ?? '';
980 - }
981 - }
982 -
983 - if (empty($host) || empty($api_key)) {
984 - return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
985 - }
986 -
987 - $base_vector_id = md5($source_url);
988 - $vectors_to_delete = array();
989 -
990 - // Add the original single-vector ID (for non-chunked content)
991 - $vectors_to_delete[] = $base_vector_id;
992 -
993 - // Pinecone /vectors/list is a GET endpoint with query-string parameters; a POST here returns a
994 - // non-200 silently and we end up only deleting the base vector, leaving chunks orphaned.
995 - $query_params = array(
996 - 'prefix' => $base_vector_id . '_chunk_',
997 - 'limit' => 100,
998 - );
999 - if (!empty($namespace)) {
1000 - $query_params['namespace'] = $namespace;
1001 - }
1002 -
1003 - $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
1004 -
1005 - // Paginate in case a URL has more than 100 chunks.
1006 - do {
1007 - $list_response = wp_remote_get($list_url, array(
1008 - 'headers' => array(
1009 - 'Api-Key' => $api_key,
1010 - 'accept' => 'application/json',
1011 - ),
1012 - 'timeout' => 30,
1013 - ));
1014 -
1015 - if (is_wp_error($list_response) || wp_remote_retrieve_response_code($list_response) !== 200) {
1016 - break;
1017 - }
1018 -
1019 - $list_data = json_decode(wp_remote_retrieve_body($list_response), true);
1020 - if (!empty($list_data['vectors'])) {
1021 - foreach ($list_data['vectors'] as $vector) {
1022 - if (isset($vector['id'])) {
1023 - $vectors_to_delete[] = $vector['id'];
1024 - }
1025 - }
1026 - }
1027 -
1028 - $next_token = $list_data['pagination']['next'] ?? '';
1029 - if (empty($next_token)) {
1030 - break;
1031 - }
1032 -
1033 - $query_params['paginationToken'] = $next_token;
1034 - $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
1035 - } while (true);
1036 -
1037 - if (empty($vectors_to_delete)) {
1038 - //error_log('[MXCHAT-CHUNK-DELETE] No vectors found to delete');
1039 - return true;
1040 - }
1041 -
1042 - //error_log('[MXCHAT-CHUNK-DELETE] Deleting ' . count($vectors_to_delete) . ' vectors from Pinecone');
1043 -
1044 - // Delete vectors
1045 - $delete_url = "https://{$host}/vectors/delete";
1046 -
1047 - $delete_body = array(
1048 - 'ids' => $vectors_to_delete
1049 - );
1050 -
1051 - if (!empty($namespace)) {
1052 - $delete_body['namespace'] = $namespace;
1053 - }
1054 -
1055 - $delete_response = wp_remote_post($delete_url, array(
1056 - 'headers' => array(
1057 - 'Api-Key' => $api_key,
1058 - 'accept' => 'application/json',
1059 - 'content-type' => 'application/json'
1060 - ),
1061 - 'body' => wp_json_encode($delete_body),
1062 - 'timeout' => 30
1063 - ));
1064 -
1065 - if (is_wp_error($delete_response)) {
1066 - return $delete_response;
1067 - }
1068 -
1069 - $response_code = wp_remote_retrieve_response_code($delete_response);
1070 - if ($response_code !== 200) {
1071 - return new WP_Error('pinecone_delete', 'Failed to delete vectors: HTTP ' . $response_code);
1072 - }
1073 -
1074 - return true;
1075 -}
1076 -
1077 -/**
1078 - * Delete all chunks for a URL from WordPress database
1079 - */
1080 -private static function delete_wordpress_chunks_by_url($source_url) {
1081 - global $wpdb;
1082 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
1083 -
1084 - // Delete all rows with this source_url (handles both chunked and non-chunked)
1085 - $result = $wpdb->delete(
1086 - $table_name,
1087 - array('source_url' => $source_url),
1088 - array('%s')
1089 - );
1090 -
1091 - if ($result === false) {
1092 - return new WP_Error('database_delete', 'Failed to delete chunks: ' . $wpdb->last_error);
1093 - }
1094 -
1095 - //error_log('[MXCHAT-CHUNK-DELETE] Deleted ' . $result . ' rows from WordPress DB');
1096 - return true;
1097 -}
1 +<?php
2 +if (!defined('ABSPATH')) {
3 + exit; // Exit if accessed directly
4 +}
5 +
6 +class MxChat_Utils {
7 +
8 +/**
9 + * UPDATED: Submit or update content (and its embedding) in the database.
10 + * Stores in Pinecone if enabled, otherwise stores in WordPress DB.
11 + *
12 + * @param string $content The content to be embedded.
13 + * @param string $source_url The source URL of the content.
14 + * @param string $api_key The API key used for generating embeddings.
15 + * @param string $vector_id Optional vector ID for Pinecone (if not provided, will use md5 of URL)
16 + * @param string $bot_id The bot ID for multi-bot support
17 + * @return bool|WP_Error True on success, WP_Error on failure
18 + */
19 +public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default') {
20 + global $wpdb;
21 + $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
22 +
23 + //error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ')');
24 + //error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
25 +
26 + // Sanitize the source URL
27 + $source_url = esc_url_raw($source_url);
28 +
29 + // Just ensure UTF-8 validity without aggressive escaping
30 + $safe_content = wp_check_invalid_utf8($content);
31 + // Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D)
32 + $safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
33 +
34 + // UPDATED: Generate the embedding using bot-specific configuration
35 + $embedding_vector = self::generate_embedding($content, $api_key, $bot_id);
36 +
37 + if (!is_array($embedding_vector)) {
38 + //error_log('[MXCHAT-DB] Error: Embedding generation failed');
39 + return new WP_Error('embedding_failed', 'Failed to generate embedding for content');
40 + }
41 +
42 + //error_log('[MXCHAT-DB] Embedding generated successfully');
43 +
44 + // UPDATED: Check if Pinecone is enabled for this specific bot
45 + if (self::is_pinecone_enabled_for_bot($bot_id)) {
46 + //error_log('[MXCHAT-DB] Pinecone is enabled for bot ' . $bot_id . ' - using Pinecone storage');
47 + // Store in Pinecone only
48 + return self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id);
49 + } else {
50 + //error_log('[MXCHAT-DB] Pinecone not enabled for bot ' . $bot_id . ' - using WordPress storage');
51 + // Store in WordPress database only
52 + $embedding_vector_serialized = maybe_serialize($embedding_vector);
53 + return self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name);
54 + }
55 +}
56 +
57 +/**
58 + * UPDATED: Check if Pinecone is enabled and properly configured for a specific bot
59 + */
60 +private static function is_pinecone_enabled_for_bot($bot_id = 'default') {
61 + // For default bot or when multi-bot is not active, use original method
62 + if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
63 + return self::is_pinecone_enabled();
64 + }
65 +
66 + // Get bot-specific Pinecone configuration
67 + $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
68 +
69 + if (empty($bot_pinecone_config)) {
70 + // Fallback to default configuration
71 + return self::is_pinecone_enabled();
72 + }
73 +
74 + $enabled_check = !empty($bot_pinecone_config['use_pinecone']) && $bot_pinecone_config['use_pinecone'];
75 + $api_key_check = !empty($bot_pinecone_config['api_key']);
76 + $host_check = !empty($bot_pinecone_config['host']);
77 +
78 + return $enabled_check && $api_key_check && $host_check;
79 +}
80 +
81 +/**
82 + * Check if Pinecone is enabled and properly configured (original method for default bot)
83 + */
84 +private static function is_pinecone_enabled() {
85 + $pinecone_options = get_option('mxchat_pinecone_addon_options');
86 +
87 + if (empty($pinecone_options)) {
88 + return false;
89 + }
90 +
91 + $enabled_check = !empty($pinecone_options['mxchat_use_pinecone']) && $pinecone_options['mxchat_use_pinecone'] !== '0';
92 + $api_key_check = !empty($pinecone_options['mxchat_pinecone_api_key']);
93 + $host_check = !empty($pinecone_options['mxchat_pinecone_host']);
94 +
95 + return $enabled_check && $api_key_check && $host_check;
96 +}
97 +
98 +/**
99 + * UPDATED: Store content in Pinecone only with bot support
100 + */
101 +private static function store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id = null, $bot_id = 'default') {
102 + //error_log('[MXCHAT-PINECONE] ===== Using Pinecone-only storage for bot ' . $bot_id . ' =====');
103 +
104 + // Get bot-specific Pinecone configuration
105 + if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
106 + $pinecone_options = get_option('mxchat_pinecone_addon_options');
107 + $api_key = $pinecone_options['mxchat_pinecone_api_key'];
108 + $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
109 + $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
110 + $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
111 + } else {
112 + $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
113 + if (empty($bot_pinecone_config)) {
114 + // Fallback to default configuration
115 + $pinecone_options = get_option('mxchat_pinecone_addon_options');
116 + $api_key = $pinecone_options['mxchat_pinecone_api_key'];
117 + $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
118 + $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
119 + $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
120 + } else {
121 + $api_key = $bot_pinecone_config['api_key'];
122 + $environment = ''; // Not used in new Pinecone API
123 + $index_name = ''; // Not used in new Pinecone API
124 + $namespace = $bot_pinecone_config['namespace'] ?? '';
125 + }
126 + }
127 +
128 + $result = self::store_in_pinecone_main(
129 + $embedding_vector,
130 + $content,
131 + $source_url,
132 + $api_key,
133 + $environment,
134 + $index_name,
135 + $vector_id,
136 + $bot_id,
137 + $namespace
138 + );
139 +
140 + if (is_wp_error($result)) {
141 + //error_log('[MXCHAT-PINECONE] Pinecone storage failed for bot ' . $bot_id . ': ' . $result->get_error_message());
142 + return $result;
143 + }
144 +
145 + //error_log('[MXCHAT-PINECONE] Pinecone storage completed successfully for bot ' . $bot_id);
146 + return true;
147 +}
148 +
149 +/**
150 + * Store content in WordPress database with progressive fallback (unchanged)
151 + */
152 +private static function store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name) {
153 + global $wpdb;
154 +
155 + //error_log('[MXCHAT-DB] ===== Using WordPress-only storage =====');
156 +
157 + // ===== FIXED: Generate unique identifier for manual content =====
158 + $original_source_url = $source_url;
159 + $is_manual_content = empty($source_url) || $source_url === '' || !filter_var($source_url, FILTER_VALIDATE_URL);
160 +
161 + if ($is_manual_content) {
162 + // Generate unique identifier for manual content to prevent overwrites
163 + $source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
164 + //error_log('[MXCHAT-DB] Generated unique ID for manual content: ' . $source_url);
165 + }
166 +
167 + // Only check for duplicates if we have a valid source URL (not manual content)
168 + $existing_id = null;
169 + if (!$is_manual_content) {
170 + $existing_id = $wpdb->get_var(
171 + $wpdb->prepare(
172 + "SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
173 + $source_url
174 + )
175 + );
176 + //error_log('[MXCHAT-DB] Checked for existing URL, found ID: ' . ($existing_id ?: 'none'));
177 + } else {
178 + //error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)');
179 + }
180 + // ===== END FIX =====
181 +
182 + // Progressive fallback mechanism for problematic content
183 + $attempt = 1;
184 + $max_attempts = 3;
185 + $current_content = $safe_content;
186 + $result = false;
187 +
188 + while ($attempt <= $max_attempts && $result === false) {
189 + try {
190 + if ($existing_id) {
191 + //error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
192 +
193 + // Update the existing row
194 + $result = $wpdb->update(
195 + $table_name,
196 + array(
197 + 'url' => $source_url,
198 + 'article_content' => $current_content,
199 + 'embedding_vector' => $embedding_vector_serialized,
200 + 'source_url' => $source_url,
201 + 'timestamp' => current_time('mysql'),
202 + ),
203 + array('id' => $existing_id),
204 + array('%s','%s','%s','%s','%s'),
205 + array('%d')
206 + );
207 + } else {
208 + //error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
209 + //error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
210 +
211 + // Insert a new row (using generated unique ID for manual content)
212 + $result = $wpdb->insert(
213 + $table_name,
214 + array(
215 + 'url' => $source_url, // Now unique for manual content
216 + 'article_content' => $current_content,
217 + 'embedding_vector' => $embedding_vector_serialized,
218 + 'source_url' => $source_url, // Now unique for manual content
219 + 'timestamp' => current_time('mysql'),
220 + ),
221 + array('%s','%s','%s','%s','%s')
222 + );
223 + }
224 +
225 + if ($result === false) {
226 + //error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . '): ' . $wpdb->last_error);
227 +
228 + // Progressively apply more aggressive sanitization on failure
229 + if ($attempt === 1) {
230 + // First fallback: Use a more aggressive character filter and shorten
231 + $current_content = preg_replace('/[^\p{L}\p{N}\s.,;:!?()-]/u', '', $current_content);
232 + $current_content = substr($current_content, 0, 50000);
233 + } else if ($attempt === 2) {
234 + // Second fallback: Keep only alphanumeric and basic punctuation, shorten further
235 + $current_content = preg_replace('/[^a-zA-Z0-9\s.,;:!?()-]/u', '', $current_content);
236 + $current_content = substr($current_content, 0, 30000);
237 + }
238 +
239 + $attempt++;
240 + }
241 + } catch (Exception $e) {
242 + //error_log('[MXCHAT-DB] Exception during database operation: ' . $e->getMessage());
243 + $attempt++;
244 + }
245 + }
246 +
247 + if ($result === false) {
248 + //error_log('[MXCHAT-DB] All database operation attempts failed');
249 + return new WP_Error('database_failed', 'Failed to store content in WordPress database after ' . $max_attempts . ' attempts');
250 + }
251 +
252 + //error_log('[MXCHAT-DB] WordPress database operation completed successfully (Attempt ' . ($attempt - 1) . ')');
253 + return true;
254 +}
255 +
256 +/**
257 + * UPDATED: Store content in Pinecone database with bot support
258 + */
259 +private static function store_in_pinecone_main($embedding_vector, $content, $url, $api_key, $environment, $index_name, $vector_id = null, $bot_id = 'default', $namespace = '') {
260 + //error_log('[MXCHAT-PINECONE-MAIN] ===== Starting Pinecone storage for bot ' . $bot_id . ' =====');
261 +
262 + // ===== UPDATED: Handle manual content with unique vector IDs =====
263 + if ($vector_id) {
264 + // Use provided vector ID
265 + //error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
266 + } elseif (!empty($url) && filter_var($url, FILTER_VALIDATE_URL)) {
267 + // For valid URLs, use URL-based ID (existing behavior)
268 + $vector_id = md5($url);
269 + //error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
270 + } else {
271 + // For manual content (empty/invalid URL), generate unique ID
272 + $vector_id = 'manual_' . time() . '_' . substr(md5($content . microtime(true)), 0, 8);
273 + //error_log('[MXCHAT-PINECONE-MAIN] Generated unique vector ID for manual content: ' . $vector_id);
274 + }
275 + // ===== END UPDATE =====
276 +
277 + // Get host from bot-specific config or fallback to default
278 + if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
279 + $options = get_option('mxchat_pinecone_addon_options');
280 + $host = $options['mxchat_pinecone_host'] ?? '';
281 + } else {
282 + $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
283 + if (!empty($bot_pinecone_config)) {
284 + $host = $bot_pinecone_config['host'] ?? '';
285 + } else {
286 + $options = get_option('mxchat_pinecone_addon_options');
287 + $host = $options['mxchat_pinecone_host'] ?? '';
288 + }
289 + }
290 +
291 + //error_log('[MXCHAT-PINECONE-MAIN] Host: ' . $host);
292 + //error_log('[MXCHAT-PINECONE-MAIN] API key length: ' . strlen($api_key));
293 + //error_log('[MXCHAT-PINECONE-MAIN] Bot ID: ' . $bot_id);
294 + //error_log('[MXCHAT-PINECONE-MAIN] Namespace: ' . $namespace);
295 +
296 + if (empty($host)) {
297 + //error_log('[MXCHAT-PINECONE-MAIN] ERROR: Host is empty');
298 + return new WP_Error('pinecone_config', 'Pinecone host is not configured. Please set the host in your bot settings.');
299 + }
300 +
301 + // ===== UPDATED: Determine content type more accurately =====
302 + $is_product = false;
303 + $content_type = 'manual'; // Default for manual content
304 +
305 + if (!empty($url) && filter_var($url, FILTER_VALIDATE_URL)) {
306 + $is_product = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
307 + $content_type = $is_product ? 'product' : 'content';
308 + }
309 +
310 + //error_log('[MXCHAT-PINECONE-MAIN] Content type: ' . $content_type);
311 + // ===== END UPDATE =====
312 +
313 + $api_endpoint = "https://{$host}/vectors/upsert";
314 + //error_log('[MXCHAT-PINECONE-MAIN] API endpoint: ' . $api_endpoint);
315 +
316 + // UPDATED: Add bot_id to metadata and handle namespace
317 + $metadata = array(
318 + 'text' => $content,
319 + 'source_url' => $url, // Can be empty for manual content
320 + 'type' => $content_type, // 'manual', 'content', or 'product'
321 + 'last_updated' => time(),
322 + 'created_at' => time(), // Add creation timestamp
323 + 'bot_id' => $bot_id // Add bot identification
324 + );
325 +
326 + $vector_data = array(
327 + 'id' => $vector_id,
328 + 'values' => $embedding_vector,
329 + 'metadata' => $metadata
330 + );
331 +
332 + $request_body = array(
333 + 'vectors' => array($vector_data)
334 + );
335 +
336 + // Add namespace if specified for multi-bot separation
337 + if (!empty($namespace)) {
338 + $request_body['namespace'] = $namespace;
339 + //error_log('[MXCHAT-PINECONE-MAIN] Using namespace: ' . $namespace);
340 + }
341 +
342 + //error_log('[MXCHAT-PINECONE-MAIN] Request body prepared (embedding dimensions: ' . count($embedding_vector) . ')');
343 +
344 + $response = wp_remote_post($api_endpoint, array(
345 + 'headers' => array(
346 + 'Api-Key' => $api_key,
347 + 'accept' => 'application/json',
348 + 'content-type' => 'application/json'
349 + ),
350 + 'body' => wp_json_encode($request_body),
351 + 'timeout' => 30,
352 + 'data_format' => 'body'
353 + ));
354 +
355 + if (is_wp_error($response)) {
356 + //error_log('[MXCHAT-PINECONE-MAIN] WordPress request error: ' . $response->get_error_message());
357 + return new WP_Error('pinecone_request', $response->get_error_message());
358 + }
359 +
360 + $response_code = wp_remote_retrieve_response_code($response);
361 + //error_log('[MXCHAT-PINECONE-MAIN] Response code: ' . $response_code);
362 +
363 + if ($response_code !== 200) {
364 + $body = wp_remote_retrieve_body($response);
365 + //error_log('[MXCHAT-PINECONE-MAIN] API error - Response body: ' . $body);
366 + return new WP_Error('pinecone_api', sprintf(
367 + 'Pinecone API error (HTTP %d): %s',
368 + $response_code,
369 + $body
370 + ));
371 + }
372 +
373 + $response_body = wp_remote_retrieve_body($response);
374 + //error_log('[MXCHAT-PINECONE-MAIN] Success response: ' . $response_body);
375 + //error_log('[MXCHAT-PINECONE-MAIN] Successfully stored in Pinecone for bot ' . $bot_id);
376 + //error_log('[MXCHAT-PINECONE-MAIN] ===== Pinecone storage complete =====');
377 +
378 + return true;
379 +}
380 +
381 +/**
382 + * UPDATED: Generate an embedding for the given text using bot-specific configuration.
383 + *
384 + * @param string $text The text to be embedded.
385 + * @param string $api_key The API key used for generating embeddings.
386 + * @param string $bot_id The bot ID for multi-bot support
387 + * @return array|null The embedding vector or null on failure.
388 + */
389 +private static function generate_embedding($text, $api_key, $bot_id = 'default') {
390 + // Get bot-specific options
391 + if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
392 + $options = get_option('mxchat_options');
393 + } else {
394 + $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
395 + $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
396 + }
397 +
398 + $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
399 +
400 + // Determine endpoint and API key based on model
401 + if (strpos($selected_model, 'voyage') === 0) {
402 + $endpoint = 'https://api.voyageai.com/v1/embeddings';
403 + $api_key = $options['voyage_api_key'] ?? '';
404 + } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
405 + $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
406 + $api_key = $options['gemini_api_key'] ?? '';
407 + } else {
408 + $endpoint = 'https://api.openai.com/v1/embeddings';
409 + // Use the bot-specific API key or fallback to passed API key
410 + $api_key = $options['api_key'] ?? $api_key;
411 + }
412 +
413 + // Prepare request body based on provider
414 + if (strpos($selected_model, 'gemini-embedding') === 0) {
415 + // Gemini API format
416 + $request_body = [
417 + 'model' => 'models/' . $selected_model,
418 + 'content' => [
419 + 'parts' => [
420 + ['text' => $text]
421 + ]
422 + ],
423 + 'outputDimensionality' => 1536
424 + ];
425 +
426 + // Prepare headers for Gemini (API key as query parameter)
427 + $endpoint .= '?key=' . $api_key;
428 + $headers = [
429 + 'Content-Type' => 'application/json'
430 + ];
431 + } else {
432 + // OpenAI/Voyage API format
433 + $request_body = [
434 + 'input' => $text,
435 + 'model' => $selected_model
436 + ];
437 +
438 + // Add output_dimension for voyage-3-large
439 + if ($selected_model === 'voyage-3-large') {
440 + $request_body['output_dimension'] = 2048;
441 + }
442 +
443 + // Prepare headers for OpenAI/Voyage
444 + $headers = [
445 + 'Content-Type' => 'application/json',
446 + 'Authorization' => 'Bearer ' . $api_key
447 + ];
448 + }
449 +
450 + $args = [
451 + 'body' => wp_json_encode($request_body),
452 + 'headers' => $headers,
453 + 'timeout' => 60,
454 + 'redirection' => 5,
455 + 'blocking' => true,
456 + 'httpversion' => '1.0',
457 + 'sslverify' => true,
458 + ];
459 +
460 + $response = wp_remote_post($endpoint, $args);
461 +
462 + if (is_wp_error($response)) {
463 + //error_log('Error generating embedding for bot ' . $bot_id . ': ' . $response->get_error_message());
464 + return null;
465 + }
466 +
467 + $response_body = json_decode(wp_remote_retrieve_body($response), true);
468 +
469 + // Handle different response formats based on provider
470 + if (strpos($selected_model, 'gemini-embedding') === 0) {
471 + // Gemini API response format
472 + if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
473 + return $response_body['embedding']['values'];
474 + } else {
475 + //error_log('Invalid response received from Gemini embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
476 + return null;
477 + }
478 + } else {
479 + // OpenAI/Voyage API response format
480 + if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
481 + return $response_body['data'][0]['embedding'];
482 + } else {
483 + //error_log('Invalid response received from embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
484 + return null;
485 + }
486 + }
487 +}
1098 488 }